How to add metadata to Gstreamer buffer in Python?

    Gstreamer is flexible plugin based data streaming framework. One of the main advantange of plugins is that developer can pass not only data buffer, but custom metadata which can describe buffer and store any information about buffer. Next information is a simple guide with code templates on how to pass any data from one plugin to another.

    Here are some possible use of metadata for Computer Vision applications:

• objects bounding boxes (list of [x, y, width, height])
• objects classes (“person”, “car”, “cat”, “dog”, … )
• objects confidences ([0, 1.0])
• keypoints
• etc.

Note: Metadata do not relies on buffer encoding, format.
To get more information about role of Metadata in Gstreamer read official docs

Problem

     It could be difficult (in some cases even impossible) to read/write metadata from gstreamer plugin implemented in Python. (Remember that Gst.Buffer isn’t writable? See “How to make Gst.Buffer writable”)

    There are some discussions from forums with problem description and with no solution:

• • Getting buffer metadata in plugin written in Python
  • How to shuttle Gst.Meta between Python and C

Solution

    Code is available here.

//
git clone https://github.com/jackersson/gst-python-hacks.git
cd gst-python-hacks
cd gst-metadata
./build

    Open how-to-write-metadata-gst-buffer.ipynb in notebook. Let’s go through each step and see how it works:

#1. Create new buffer

//
buffer = Gst.Buffer.new_wrapped(b"lifestyletransfer")

#2. Write custom meta with text field

//
write_meta(buffer, description="Turotial 'How to write metadata in Python Gstreamer plugin'")

#3. Check that meta exists

//
buffer_info_meta = get_meta(buffer)
print(buffer_info_meta.description.decode("utf-8"))
->
Turotial 'How to write metadata in Python Gstreamer plugin'

#4. Remove meta

//
is_ok = remove_meta(buffer)
print(is_ok)
->
True

Explanation

    There are specific requirements from official documentation for handling buffer’s metadata. All those requirements could be combined in C-code template (common for all custom metadata you might want to use).
    Have a look at simple example on how to use C-template to create your own Metadata.

#1. Define Metadata Structure

//
// Structure that holds "description" field to 
// store custom text data
struct GstBufferInfo {
    gchar* description;
};

// Custom structure to be passed between Gstreamer plugins
struct GstBufferInfoMeta {

    // Required as it is base structure for metadata
    // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/gstreamer-GstMeta.html
    GstMeta meta;

    // Custom fields
    GstBufferInfo info;
};

//
GType gst_buffer_info_meta_api_get_type(void)
{
    static const gchar *tags[] = {NULL};
    static volatile GType type;
    if (g_once_init_enter (&type)) {
        GType _type = gst_meta_api_type_register("GstBufferInfoMetaAPI", tags);
        g_once_init_leave(&type, _type);
    }
    return type;
}

//
static gboolean gst_buffer_info_meta_init(GstMeta *meta, gpointer params, GstBuffer *buffer)
{
    GstBufferInfoMeta *gst_buffer_info_meta = (GstBufferInfoMeta*)meta;
    gst_buffer_info_meta->info.description = "";
    return TRUE;
}

//
static gboolean gst_buffer_info_meta_transform(GstBuffer *transbuf, GstMeta *meta, 
                                               GstBuffer *buffer,
                                               GQuark type, gpointer data)
{
    GstBufferInfoMeta *gst_buffer_info_meta = (GstBufferInfoMeta *)meta;
    gst_buffer_add_buffer_info_meta(transbuf, &(gst_buffer_info_meta->info));
    return TRUE;
}

//

const GstMetaInfo *gst_buffer_info_meta_get_info(void)
{
    static const GstMetaInfo *gst_buffer_info_meta_info = NULL;
 
    if (g_once_init_enter (&gst_buffer_info_meta_info)) {

        // Explanation of fields
        // https://gstreamer.freedesktop.org/documentation/design/meta.html#gstmeta1
        const GstMetaInfo *meta = gst_meta_register (GST_BUFFER_INFO_META_API_TYPE, /* api type */
            "GstBufferInfoMeta", /* implementation type */
            sizeof(GstBufferInfoMeta), /* size of the structure */
            gst_buffer_info_meta_init,
            (GstMetaFreeFunction) NULL,
            gst_buffer_info_meta_transform);
       g_once_init_leave (&gst_buffer_info_meta_info, meta);
    }
    return gst_buffer_info_meta_info;
}

//
GstBufferInfoMeta* meta = (GstBufferInfoMeta*)gst_buffer_get_meta((buffer), GST_BUFFER_INFO_META_API_TYPE);

//
gst_buffer_info_meta = (GstBufferInfoMeta *) gst_buffer_add_meta (buffer, GST_BUFFER_INFO_META_INFO, NULL);

// copy fields to buffer's meta
if (buffer_info->description!=NULL)
{
    gst_buffer_info_meta->info.description = malloc(strlen(buffer_info->description) + 1);
    strcpy(gst_buffer_info_meta->info.description, buffer_info->description);
}

//
GstBufferInfoMeta* meta = (GstBufferInfoMeta*)gst_buffer_get_meta((buffer), GST_BUFFER_INFO_META_API_TYPE);

// https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/GstBuffer.html#gst-buffer-remove-meta
gboolean is_ok = gst_buffer_remove_meta(buffer, &meta->meta);

//

gst_buffer_info_meta.c
gst_buffer_info_meta.h

//

./build.sh

    Library file libgst_buffer_info_meta.so is now located in build/ folder.

    Now we can call C-functions from shared library using ctypes and Python. Let use next step-by-step guide

#1. Load shared lib

    To load libgst_buffer_info_meta.so library use:

# 

clib = CDLL("build/libgst_buffer_info_meta.so")

#2. Define Python version GstBufferInfo Structure (with ctypes) that corresponds to C-version to GstBufferInfoMeta

# 

class GstBufferInfo(Structure):
    _fields_ = [("description", c_char_p)]

GstBufferInfoPtr = POINTER(GstBufferInfo)

# 

clib.gst_buffer_add_buffer_info_meta.argtypes = [c_void_p, GstBufferInfoPtr]
clib.gst_buffer_add_buffer_info_meta.restype = c_void_p

clib.gst_buffer_get_buffer_info_meta.argtypes = [c_void_p]
clib.gst_buffer_get_buffer_info_meta.restype = GstBufferInfoPtr

clib.gst_buffer_remove_buffer_info_meta.argtypes = [c_void_p]
clib.gst_buffer_remove_buffer_info_meta.restype = c_bool

#

def write_meta(buffer, description):   
    meta = GstBufferInfo()
    meta.description = description.encode("utf-8")
    clib.gst_buffer_add_buffer_info_meta(hash(buffer), meta)

def get_meta(buffer):   
    res = clib.gst_buffer_get_buffer_info_meta(hash(buffer))
    return res.contents

def remove_meta(buffer):
    return clib.gst_buffer_remove_buffer_info_meta(hash(buffer))

    Previous functions are used in how-to-write-metadata-gst-buffer.ipynb. Now you can extend Metadata structure and use in your applications. Hope everything works as expected, if not – leave a comment 😉

    Additional readings:

• How to design Metadata for Gstreamer