Add generic glsvgoverlay element
glsvgoverlay is similar to the old glsvgoverlaysink but is a
filter instead of a sink. That means that the overlays are
rendered on the frame and passed downstream instead of overlays
being drawn on top of the frame in the sink. This enables passing
the output to e.g. a video encoder and not just HDMI.
Two modes are supported set by the bool property 'sync'. When
true the element will require a SVG overlay with the same PTS
(presentation timestamp) as the frame; no incoming video frame
will be rendered without the corresponding overlay. The sync
property set to false means the element will draw the last
available SVG overlay over a frame, which allows for higher
throughput and potentially visual fidelity at the expense of the
occasional de-synced overlay and video frame.
Bug: 161188385
Change-Id: Ice1047a0ebb1a3ae9f12f178e19a0cc9f22d1720
diff --git a/debian/control b/debian/control
index a393b05..bcace03 100644
--- a/debian/control
+++ b/debian/control
@@ -2,7 +2,7 @@
Maintainer: Coral <coral-support@google.com>
Section: python
Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9), libgstreamer-plugins-base1.0-dev, pkg-config
+Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9), libgstreamer-plugins-base1.0-dev, pkg-config, librsvg2-dev, libcairo2-dev
Standards-Version: 3.9.8
Homepage: https://coral.ai/
diff --git a/debian/rules b/debian/rules
index 187cf7a..2d838db 100755
--- a/debian/rules
+++ b/debian/rules
@@ -4,9 +4,10 @@
PLUGINSDIR := $(shell pkg-config --variable=pluginsdir gstreamer-1.0)
PY_DESTDIR := debian/python3-edgetpuvision/$(PLUGINSDIR)/python
SO_DESTDIR := debian/gstreamer1.0-plugins-coral/$(PLUGINSDIR)
-SO_SRCS := plugins/gstglbox.c plugins/gstcoral.c
+SO_SRCS := $(addprefix plugins/, gstcoral.c gstglbox.c gstglsvgoverlay.c)
SO_DEST := $(SO_DESTDIR)/libgstcoral.so
-SO_DEPS := gstreamer-plugins-base-1.0 gstreamer-video-1.0 gstreamer-gl-1.0
+SO_DEPS := gstreamer-plugins-base-1.0 gstreamer-video-1.0 gstreamer-gl-1.0 \
+ librsvg-2.0 cairo
export PYBUILD_NAME=edgetpuvision
@@ -19,4 +20,5 @@
install -g 0 -o 0 plugins/python/*.py $(PY_DESTDIR)
install -d $(SO_DESTDIR)
- $(DEB_HOST_GNU_TYPE)-gcc `$(DEB_HOST_GNU_TYPE)-pkg-config --libs --cflags $(SO_DEPS)` $(CFLAGS) -shared -o $(SO_DEST) -DCORAL_VERSION=$(DEB_VERSION) $(SO_SRCS)
+ $(DEB_HOST_GNU_TYPE)-gcc `$(DEB_HOST_GNU_TYPE)-pkg-config --libs --cflags $(SO_DEPS)` \
+ $(CFLAGS) -shared -o $(SO_DEST) -DCORAL_VERSION=$(DEB_VERSION) $(SO_SRCS)
diff --git a/plugins/gstcoral.c b/plugins/gstcoral.c
index 7b6cac6..07be7cd 100644
--- a/plugins/gstcoral.c
+++ b/plugins/gstcoral.c
@@ -19,6 +19,7 @@
#endif
#include "gstglbox.h"
+#include "gstglsvgoverlay.h"
#define CORAL_LICENSE GST_LICENSE_UNKNOWN /* Apache not supported */
#define ORIGIN "https://coral.ai"
@@ -33,6 +34,11 @@
return FALSE;
}
+ if (!gst_element_register (plugin, "glsvgoverlay",
+ GST_RANK_NONE, gst_gl_svg_overlay_get_type ())) {
+ return FALSE;
+ }
+
return TRUE;
}
diff --git a/plugins/gstglsvgoverlay.c b/plugins/gstglsvgoverlay.c
new file mode 100644
index 0000000..7dceb68
--- /dev/null
+++ b/plugins/gstglsvgoverlay.c
@@ -0,0 +1,690 @@
+/*
+ * # Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <cairo.h>
+#include <librsvg/rsvg.h>
+
+#include <gst/gl/gstglfuncs.h>
+#include "gstglsvgoverlay.h"
+
+#define GST_CAT_DEFAULT gst_gl_svg_overlay_debug
+GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
+
+#define DEBUG_INIT \
+ GST_DEBUG_CATEGORY_INIT (gst_gl_svg_overlay_debug, \
+ "glsvgoverlay", 0, "glsvgoverlay element");
+#define gst_gl_svg_overlay_parent_class parent_class
+G_DEFINE_TYPE_WITH_CODE (GstGLSvgOverlay, gst_gl_svg_overlay,
+ GST_TYPE_GL_FILTER, DEBUG_INIT);
+
+enum
+{
+ PROP_0,
+ PROP_DATA,
+ PROP_SVG,
+ PROP_SYNC,
+};
+
+#define DEFAULT_PROP_SYNC TRUE
+
+static void gst_gl_svg_overlay_set_property (GObject * object, guint prop_id,
+ const GValue * value, GParamSpec * pspec);
+static void gst_gl_svg_overlay_get_property (GObject * object, guint prop_id,
+ GValue * value, GParamSpec * pspec);
+
+static GstStateChangeReturn gst_gl_svg_overlay_change_state (
+ GstElement * element, GstStateChange transition);
+static gboolean gst_gl_svg_overlay_gl_start (GstGLBaseFilter * base_filter);
+static void gst_gl_svg_overlay_gl_stop (GstGLBaseFilter * base_filter);
+static gboolean gst_gl_svg_overlay_gl_set_caps (GstGLBaseFilter * base_filter,
+ GstCaps * incaps, GstCaps * outcaps);
+static GstFlowReturn gst_gl_svg_overlay_transform (GstBaseTransform * bt,
+ GstBuffer * inbuf, GstBuffer * outbuf);
+static gboolean gst_gl_svg_overlay_filter_texture (GstGLFilter * filter,
+ GstGLMemory * in_tex, GstGLMemory * out_tex);
+static gboolean gst_gl_svg_overlay_set_svg (GstGLSvgOverlay * overlay,
+ const gchar * svg, GstClockTime pts);
+static void gst_gl_svg_overlay_finalize (GObject * object);
+static void gst_gl_svg_overlay_task (gpointer data, gpointer user_data);
+static void gst_gl_svg_overlay_task ();
+
+enum
+{
+ SIGNAL_SET_SVG,
+ LAST_SIGNAL
+};
+
+static guint gst_gl_svg_overlay_signals[LAST_SIGNAL] = { 0 };
+
+static void
+gst_gl_svg_overlay_class_init (GstGLSvgOverlayClass * klass)
+{
+ gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
+
+ G_OBJECT_CLASS (klass)->set_property = gst_gl_svg_overlay_set_property;
+ G_OBJECT_CLASS (klass)->get_property = gst_gl_svg_overlay_get_property;
+ G_OBJECT_CLASS (klass)->finalize = gst_gl_svg_overlay_finalize;
+
+ /* Aliased property for compability reasons. */
+ g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_DATA,
+ g_param_spec_string ("data", "data", "SVG data", "",
+ G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS));
+ g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_SVG,
+ g_param_spec_string ("svg", "svg", "SVG data", "",
+ G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS));
+
+ g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_SYNC,
+ g_param_spec_boolean ("sync", "sync",
+ "Require one SVG overlay per frame, or use the last"
+ "available.",
+ DEFAULT_PROP_SYNC, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+ gst_element_class_set_metadata (GST_ELEMENT_CLASS (klass),
+ "OpenGL SVG overlay",
+ "Filter/Effect/Video",
+ "Overlay GL video texture with a SVG image",
+ "Coral <coral-support@google.com>");
+ GST_ELEMENT_CLASS (klass)->change_state = gst_gl_svg_overlay_change_state;
+
+ gst_gl_svg_overlay_signals[SIGNAL_SET_SVG] =
+ g_signal_new ("set-svg", G_TYPE_FROM_CLASS (klass),
+ G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION,
+ G_STRUCT_OFFSET (GstGLSvgOverlayClass, set_svg),
+ NULL, NULL, NULL,
+ G_TYPE_BOOLEAN, 2, G_TYPE_STRING, G_TYPE_UINT64);
+ klass->set_svg = gst_gl_svg_overlay_set_svg;
+
+ GST_BASE_TRANSFORM_CLASS (klass)->passthrough_on_same_caps = FALSE;
+ GST_BASE_TRANSFORM_CLASS (klass)->transform = gst_gl_svg_overlay_transform;
+
+ GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_svg_overlay_gl_start;
+ GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_svg_overlay_gl_stop;
+ GST_GL_BASE_FILTER_CLASS (klass)->gl_set_caps =
+ gst_gl_svg_overlay_gl_set_caps;
+ GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
+ GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
+ GST_GL_FILTER_CLASS (klass)->filter_texture =
+ gst_gl_svg_overlay_filter_texture;
+}
+
+static void
+gst_gl_svg_overlay_init (GstGLSvgOverlay * overlay)
+{
+ GST_DEBUG_OBJECT (overlay, "init");
+ overlay->sync = DEFAULT_PROP_SYNC;
+ overlay->num_tasks = 0;
+ overlay->started = FALSE;
+ overlay->shader = NULL;
+ overlay->thread_pool = NULL;
+ overlay->svg_queue = NULL;
+ overlay->gl_queue = NULL;
+ overlay->next_pts = 0;
+ overlay->current = NULL;
+ g_mutex_init (&overlay->mutex);
+ g_cond_init (&overlay->cond);
+}
+
+static void
+gst_gl_svg_overlay_finalize (GObject * object)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (object);
+
+ GST_DEBUG_OBJECT (overlay, "finalize");
+
+ g_mutex_clear (&overlay->mutex);
+ g_cond_clear (&overlay->cond);
+
+ G_OBJECT_CLASS (parent_class)->finalize (object);
+}
+
+static void
+gst_gl_svg_overlay_set_property (GObject * object, guint prop_id,
+ const GValue * value, GParamSpec * pspec)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (object);
+
+ switch (prop_id) {
+ case PROP_DATA:
+ case PROP_SVG:
+ gst_gl_svg_overlay_set_svg (GST_GL_SVG_OVERLAY (object),
+ g_value_get_string (value), GST_CLOCK_TIME_NONE);
+ break;
+ case PROP_SYNC:
+ g_mutex_lock (&overlay->mutex);
+ overlay->sync = g_value_get_boolean (value);
+ g_mutex_unlock (&overlay->mutex);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ break;
+ }
+}
+
+static void
+gst_gl_svg_overlay_get_property (GObject * object, guint prop_id,
+ GValue * value, GParamSpec * pspec)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (object);
+
+ switch (prop_id) {
+ case PROP_SYNC:
+ g_mutex_lock (&overlay->mutex);
+ g_value_set_boolean (value, overlay->sync);
+ g_mutex_unlock (&overlay->mutex);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ break;
+ }
+}
+
+static gint _task_data_compare (gconstpointer a, gconstpointer b,
+ gpointer user_data)
+{
+ const TaskData *t1 = a;
+ const TaskData *t2 = b;
+
+ if (t1->pts < t2->pts) {
+ return -1;
+ } else if (t1->pts > t2->pts) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
+static void
+_task_data_map (TaskData * tdata)
+{
+ gboolean ret;
+ if (tdata->map.data) {
+ return;
+ }
+
+ ret = gst_buffer_map (tdata->buf, &tdata->map, GST_MAP_READ | GST_MAP_WRITE);
+ g_assert (ret);
+}
+
+static void
+_task_data_unmap (TaskData * tdata)
+{
+ if (!tdata->map.data) {
+ return;
+ }
+
+ gst_buffer_unmap (tdata->buf, &tdata->map);
+ memset (&tdata->map, 0, sizeof (tdata->map));
+}
+
+static void
+_task_data_free_gl (GstGLContext * context, gpointer data)
+{
+ TaskData *tdata = data;
+ const GstGLFuncs *gl = context->gl_vtable;
+
+ if (!tdata || !tdata->tex) {
+ return;
+ }
+
+ gl->DeleteTextures (1, &tdata->tex);
+ tdata->tex = 0;
+}
+
+static void
+_task_data_free (gpointer data)
+{
+ TaskData *tdata = data;
+
+ if (!tdata) {
+ return;
+ }
+
+ _task_data_unmap (tdata);
+ gst_buffer_replace (&tdata->buf, NULL);
+ gst_gl_context_thread_add (tdata->overlay->context, _task_data_free_gl,
+ tdata);
+ g_free (tdata->svg);
+ g_free (tdata);
+}
+
+static void
+_task_data_upload_gl (GstGLContext * context, gpointer data)
+{
+ TaskData * tdata = data;
+ GstGLSvgOverlay *overlay = tdata->overlay;
+ GstGLFilter *filter = GST_GL_FILTER (overlay);
+ const GstGLFuncs *gl = context->gl_vtable;
+
+ /* TODO: handle non packed stride */
+ /* TODO: TexImage2D in a separate derived gl context */
+
+ gl->GenTextures (1, &tdata->tex);
+ gl->BindTexture (GL_TEXTURE_2D, tdata->tex);
+ gl->TexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, overlay->width, overlay->height, 0,
+ GL_RGBA, GL_UNSIGNED_BYTE, tdata->map.data);
+ gl->GenerateMipmap (GL_TEXTURE_2D);
+}
+
+static void
+_task_push_locked (GstGLSvgOverlay *overlay, gpointer data)
+{
+ g_thread_pool_push (overlay->thread_pool, data, NULL);
+ overlay->num_tasks++;
+}
+
+static GstStateChangeReturn
+gst_gl_svg_overlay_change_state (GstElement * element,
+ GstStateChange transition)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (element);
+
+ switch (transition) {
+ case GST_STATE_CHANGE_READY_TO_PAUSED:
+ g_mutex_lock (&overlay->mutex);
+ overlay->svg_queue = g_queue_new ();
+ overlay->gl_queue = g_queue_new ();
+ overlay->thread_pool = g_thread_pool_new (gst_gl_svg_overlay_task,
+ overlay, g_get_num_processors (), FALSE, NULL);
+ overlay->started = TRUE;
+ g_mutex_unlock (&overlay->mutex);
+ break;
+ case GST_STATE_CHANGE_PAUSED_TO_READY:
+ g_mutex_lock (&overlay->mutex);
+ overlay->started = FALSE;
+ g_cond_broadcast (&overlay->cond);
+ while (overlay->num_tasks) {
+ g_cond_wait (&overlay->cond, &overlay->mutex);
+ }
+ g_thread_pool_free (overlay->thread_pool, FALSE, FALSE);
+ overlay->thread_pool = NULL;
+ g_queue_free_full (overlay->svg_queue, _task_data_free);
+ overlay->svg_queue = NULL;
+ g_queue_free_full (overlay->gl_queue, _task_data_free);
+ overlay->gl_queue = NULL;
+ _task_data_free (overlay->current);
+ overlay->current = NULL;
+ g_mutex_unlock (&overlay->mutex);
+ break;
+ default:
+ break;
+ }
+
+ return GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
+}
+
+static gboolean
+gst_gl_svg_overlay_gl_start (GstGLBaseFilter * base_filter)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (base_filter);
+ GstGLFilter *filter = GST_GL_FILTER (base_filter);
+
+ GST_DEBUG_OBJECT (overlay, "gl_start");
+
+ /* TODO: create a shared context for upload */
+ overlay->context = gst_object_ref (GST_OBJECT (base_filter->context));
+ overlay->shader = gst_gl_shader_new_default (overlay->context, NULL);
+
+ filter->draw_attr_position_loc =
+ gst_gl_shader_get_attribute_location (overlay->shader, "a_position");
+ filter->draw_attr_texture_loc =
+ gst_gl_shader_get_attribute_location (overlay->shader, "a_texcoord");
+
+ return GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base_filter);
+}
+
+static void
+gst_gl_svg_overlay_gl_stop (GstGLBaseFilter * base_filter)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (base_filter);
+ GST_DEBUG_OBJECT (overlay, "gl_stop");
+
+ _task_data_free (overlay->current);
+ overlay->current = NULL;
+
+ if (overlay->shader) {
+ gst_object_unref (overlay->shader);
+ overlay->shader = NULL;
+ }
+
+ GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base_filter);
+
+ gst_object_unref (GST_OBJECT (overlay->context));
+ overlay->context = NULL;
+}
+
+static gboolean
+gst_gl_svg_overlay_gl_set_caps (GstGLBaseFilter * base_filter, GstCaps * incaps,
+ GstCaps * outcaps)
+{
+ GstGLFilter *filter = GST_GL_FILTER (base_filter);
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (base_filter);
+ gboolean ret;
+
+ ret = GST_GL_BASE_FILTER_CLASS (parent_class)->gl_set_caps (base_filter,
+ incaps, outcaps);
+
+ overlay->width = GST_VIDEO_INFO_WIDTH (&filter->out_info);
+ overlay->height = GST_VIDEO_INFO_HEIGHT (&filter->out_info);
+ overlay->stride = cairo_format_stride_for_width (CAIRO_FORMAT_ARGB32,
+ overlay->width);
+
+ GST_DEBUG_OBJECT (filter, "%ux%u px, stride %u bytes", overlay->width,
+ overlay->height, overlay->stride);
+
+ if (overlay->stride != overlay->width * 4) {
+ /* TODO: stride */
+ GST_ERROR_OBJECT (overlay, "Unsupported stride");
+ return FALSE;
+ }
+
+ return ret;
+}
+
+static void
+gst_gl_svg_overlay_task (gpointer data, gpointer user_data)
+{
+ GstGLFilter *filter = GST_GL_FILTER (user_data);
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (user_data);
+ TaskData *tdata = NULL;
+ GstClockTime start = gst_util_get_timestamp ();
+ gdouble elapsed_ms;
+
+ /* TODO: Render SVG to dma-bufs when supported */
+
+#define LOG_ELAPSED(s) \
+do { \
+ elapsed_ms = ((gdouble) gst_util_get_timestamp () - start) / GST_MSECOND; \
+ GST_LOG_OBJECT (overlay, "%s: %.2f", s, elapsed_ms); \
+} while (0)
+
+#define TASK_DONE(push) \
+do { \
+ g_mutex_lock (&overlay->mutex); \
+ if (push) { \
+ if (overlay->started) { \
+ _task_push_locked (overlay, tdata); \
+ } else { \
+ _task_data_free (tdata); \
+ } \
+ }\
+ overlay->num_tasks--; \
+ g_cond_broadcast (&overlay->cond); \
+ g_mutex_unlock (&overlay->mutex); \
+ return; \
+} while (0)
+
+ if (data == overlay->svg_queue) {
+ g_mutex_lock (&overlay->mutex);
+ if (!overlay->sync) {
+ /* Draw only the latest SVG overlay, drop older. */
+ while (g_queue_get_length (overlay->svg_queue) > 1) {
+ tdata = g_queue_pop_head (overlay->svg_queue);
+ _task_data_free (tdata);
+ tdata = NULL;
+ }
+ }
+ tdata = g_queue_pop_head (overlay->svg_queue);
+
+ g_mutex_unlock (&overlay->mutex);
+
+ TASK_DONE (TRUE);
+ }
+
+ tdata = data;
+
+ if (tdata->op == OP_ALLOC) {
+ /* TODO: Buffer pool. */
+ const gsize size = GST_VIDEO_INFO_WIDTH (&filter->out_info) *
+ GST_VIDEO_INFO_HEIGHT (&filter->out_info) * 4; /* BGRA 4 bpp. */
+ tdata->buf = gst_buffer_new_allocate (NULL, size, NULL);
+ tdata->op = OP_DRAW;
+ LOG_ELAPSED ("alloc");
+ TASK_DONE(tdata);
+ }
+
+ if (tdata->op == OP_DRAW) {
+ cairo_t *cairo;
+ cairo_surface_t *surface;
+ RsvgHandle *handle;
+ GError *err = NULL;
+ gboolean ret;
+
+ _task_data_map (tdata);
+
+ memset (tdata->map.data, 0, tdata->map.size);
+ surface = cairo_image_surface_create_for_data (tdata->map.data,
+ CAIRO_FORMAT_ARGB32, overlay->width, overlay->height, overlay->stride);
+ g_assert (cairo_surface_status (surface) == CAIRO_STATUS_SUCCESS);
+ cairo = cairo_create (surface);
+ g_assert (cairo_status (cairo) == CAIRO_STATUS_SUCCESS);
+
+ handle = rsvg_handle_new_from_data (tdata->svg, strlen (tdata->svg), &err);
+ if (err) {
+ GST_ERROR_OBJECT (overlay, "Unable to render SVG: %s", err->message);
+ g_error_free (err);
+ }
+ g_assert (handle);
+ ret = rsvg_handle_render_cairo (handle, cairo);
+ g_assert (ret);
+
+ rsvg_handle_close (handle, NULL);
+ g_object_unref (handle);
+ cairo_surface_flush (surface);
+ cairo_surface_destroy (surface);
+ cairo_destroy (cairo);
+
+ tdata->op = OP_UPLOAD;
+ LOG_ELAPSED ("draw");
+ TASK_DONE (TRUE);
+ }
+
+ if (tdata->op == OP_UPLOAD) {
+ gst_gl_context_thread_add (tdata->overlay->context, _task_data_upload_gl,
+ tdata);
+ _task_data_unmap (tdata);
+ gst_buffer_replace (&tdata->buf, NULL);
+ tdata->op = OP_READY;
+
+ g_mutex_lock (&overlay->mutex);
+ g_queue_push_tail (overlay->gl_queue, tdata);
+ g_queue_sort (overlay->gl_queue, _task_data_compare, NULL);
+ g_cond_broadcast (&overlay->cond);
+ g_mutex_unlock (&overlay->mutex);
+ LOG_ELAPSED ("upload");
+ TASK_DONE (FALSE);
+ }
+
+ g_assert_not_reached ();
+
+#undef LOG_ELAPSED
+#undef TASK_DONE
+}
+
+static gboolean
+_overlay_ready (GstGLSvgOverlay * overlay, GstClockTime pts)
+{
+ gint i;
+
+ for (i = 0; i < g_queue_get_length (overlay->gl_queue); i++) {
+ TaskData *tdata = g_queue_peek_nth (overlay->gl_queue, i);
+ if (tdata && tdata->pts == pts && tdata->op == OP_READY) {
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static GstFlowReturn
+gst_gl_svg_overlay_transform (GstBaseTransform * bt,
+ GstBuffer * inbuf, GstBuffer * outbuf)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (bt);
+
+ g_mutex_lock (&overlay->mutex);
+ if (overlay->sync) {
+ /* In this mode we require an overlay per frame. Block this thread
+ * while the texture is being prepared in the GL thread before
+ * transferring execution there. Don't block the GL thread. */
+ while (overlay->started && !_overlay_ready (overlay,
+ GST_BUFFER_PTS (inbuf))) {
+ g_cond_wait (&overlay->cond, &overlay->mutex);
+ }
+ if (!overlay->started) {
+ g_mutex_unlock (&overlay->mutex);
+ return GST_FLOW_FLUSHING;
+ }
+ } else {
+ /* Async drawing of SVG was kicked off when property was set.
+ * When composing the final output the latest ready overlay will
+ * be used, if any.
+ */
+ }
+ overlay->next_pts = GST_BUFFER_PTS (inbuf);
+ g_mutex_unlock (&overlay->mutex);
+
+ return GST_BASE_TRANSFORM_CLASS (parent_class)->transform (bt, inbuf, outbuf);
+}
+
+static gboolean
+gst_gl_svg_overlay_set_svg (GstGLSvgOverlay * overlay, const gchar * svg,
+ GstClockTime pts)
+{
+ TaskData *tdata;
+
+ g_mutex_lock (&overlay->mutex);
+ if (!overlay->started) {
+ GST_WARNING_OBJECT (overlay, "Wrong state, dropping SVG overlay");
+ g_mutex_unlock (&overlay->mutex);
+
+ }
+
+ tdata = g_malloc0 (sizeof(TaskData));
+ tdata->overlay = overlay;
+ tdata->op = OP_ALLOC;
+ tdata->svg = g_strdup (svg);
+ tdata->pts = pts;
+ g_queue_push_tail (overlay->svg_queue, tdata);
+ _task_push_locked (overlay, overlay->svg_queue);
+
+ g_mutex_unlock (&overlay->mutex);
+
+ return TRUE;
+}
+
+static TaskData *
+gst_gl_svg_overlay_next (GstGLSvgOverlay * overlay)
+{
+ gint i;
+ GQueue *q;
+ GstClockTime pts;
+ TaskData *tdata;
+
+ g_mutex_lock (&overlay->mutex);
+
+ q = overlay->gl_queue;
+ pts = overlay->next_pts;
+
+ if (overlay->sync) {
+ /* Get overlay with matching pts, drop older. */
+ while (!g_queue_is_empty(q) &&
+ ((TaskData*) g_queue_peek_tail (q))->pts < pts) {
+ _task_data_free (g_queue_pop_head (q));
+ }
+
+ for (i = 0; i < g_queue_get_length (q); i++) {
+ TaskData *tmp = g_queue_peek_nth (q, i);
+ if (tmp && tmp->pts == pts) {
+ tdata = g_queue_pop_nth (q, i);
+ break;
+ }
+ }
+ g_assert (tdata);
+ } else {
+ /* Get the latest overlay, drop older. */
+ tdata = g_queue_pop_head (q);
+ while (!g_queue_is_empty(q)) {
+ _task_data_free (g_queue_pop_head (q));
+ }
+ }
+
+ g_mutex_unlock (&overlay->mutex);
+
+ return tdata;
+}
+
+static gboolean
+gst_gl_svg_overlay_draw_gl (GstGLFilter * filter, GstGLMemory * in_tex,
+ gpointer user_data)
+{
+ GstGLSvgOverlay *overlay = GST_GL_SVG_OVERLAY (user_data);
+ GstGLContext *context = GST_GL_BASE_FILTER (filter)->context;
+ const GstGLFuncs *gl = context->gl_vtable;
+ TaskData *next, *old = NULL;
+ guint tex_id, o_tex_id = 0;
+
+ next = gst_gl_svg_overlay_next (overlay);
+ if (next) {
+ _task_data_free (overlay->current);
+ overlay->current = next;
+ }
+
+ tex_id = gst_gl_memory_get_texture_id (in_tex);
+ if (overlay->current) {
+ o_tex_id = overlay->current->tex;
+ }
+
+ g_return_val_if_fail (overlay->shader, FALSE);
+
+ gl->ClearColor (0.0, 0.0, 0.0, 1.0);
+ gl->Clear (GL_COLOR_BUFFER_BIT);
+
+ gst_gl_shader_use (overlay->shader);
+ gst_gl_shader_set_uniform_1i (overlay->shader, "tex", 0);
+ gl->ActiveTexture (GL_TEXTURE0);
+ gl->BindTexture (GL_TEXTURE_2D, tex_id);
+ gst_gl_filter_draw_fullscreen_quad (filter);
+
+ if (o_tex_id) {
+ gst_gl_shader_use (overlay->shader);
+ gst_gl_shader_set_uniform_1i (overlay->shader, "tex", 0);
+ gl->ActiveTexture (GL_TEXTURE0);
+ gl->BindTexture (GL_TEXTURE_2D, o_tex_id);
+ gl->Enable (GL_BLEND);
+ gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+ gl->BlendEquation (GL_FUNC_ADD);
+ gst_gl_filter_draw_fullscreen_quad (filter);
+ gl->Disable (GL_BLEND);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+gst_gl_svg_overlay_filter_texture (GstGLFilter * filter, GstGLMemory * in_tex,
+ GstGLMemory * out_tex)
+{
+ if (gst_gl_context_get_gl_api (GST_GL_BASE_FILTER (filter)->context)) {
+ gst_gl_filter_render_to_target (filter, in_tex, out_tex,
+ gst_gl_svg_overlay_draw_gl, filter);
+ }
+
+ return TRUE;
+}
diff --git a/plugins/gstglsvgoverlay.h b/plugins/gstglsvgoverlay.h
new file mode 100644
index 0000000..796107b
--- /dev/null
+++ b/plugins/gstglsvgoverlay.h
@@ -0,0 +1,86 @@
+/*
+ * # Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _GST_GLSVGOVERLAY_H
+#define _GST_GLSVGOVERLAY_H
+
+#include <gst/gst.h>
+
+#include <gst/gl/gstglfilter.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_GL_SVG_OVERLAY (gst_gl_svg_overlay_get_type())
+#define GST_GL_SVG_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_SVG_OVERLAY,GstGLSvgOverlay))
+#define GST_IS_GL_SVG_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_SVG_OVERLAY))
+#define GST_GL_SVG_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass) ,GST_TYPE_GL_SVG_OVERLAY,GstGLSvgOverlayClass))
+#define GST_IS_GL_SVG_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass) ,GST_TYPE_GL_SVG_OVERLAY))
+#define GST_GL_SVG_OVERLAY_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj) ,GST_TYPE_GL_SVG_OVERLAY,GstGLSvgOverlayClass))
+
+typedef struct _GstGLSvgOverlay GstGLSvgOverlay;
+typedef struct _GstGLSvgOverlayClass GstGLSvgOverlayClass;
+typedef struct _TaskData TaskData;
+
+struct _GstGLSvgOverlay
+{
+ GstGLFilter filter;
+
+ GstGLContext *context;
+ GMutex mutex;
+ GCond cond;
+ GstGLShader *shader;
+ GThreadPool *thread_pool;
+ GQueue *svg_queue;
+ GQueue *gl_queue;
+ GstClockTime next_pts;
+ guint num_tasks;
+ TaskData *current;
+ gboolean sync;
+ gboolean started;
+ guint width;
+ guint height;
+ guint stride;
+};
+
+struct _GstGLSvgOverlayClass
+{
+ GstGLFilterClass filter_class;
+ gboolean (*set_svg) (GstGLSvgOverlay *overlay, const gchar *svg, GstClockTime pts);
+};
+
+enum TaskOp
+{
+ OP_ALLOC,
+ OP_DRAW,
+ OP_UPLOAD,
+ OP_READY,
+};
+
+struct _TaskData {
+ GstGLSvgOverlay *overlay;
+ enum TaskOp op;
+ gchar *svg;
+ GstClockTime pts;
+ GstBuffer *buf;
+ GstMapInfo map;
+ guint tex;
+};
+
+GType gst_gl_svg_overlay_get_type (void);
+
+G_END_DECLS
+
+#endif /* _GST_GLSVGOVERLAY_H */
