Respond to appsink CONTEXT queries
On GStreamer 1.14 we get different GL context in the two tee branches
which is one reason why the inference branch sees solid black frames
(i.e. all zeroes). Since CONTEXT queries now aren't propagated from
one tee branch to others they never reach glimagesink so it can't
share its context. We thus respond with said context when the query
reaches the appsink, so that only one GL context is shared between
all GL elements. If multiple contexts are used there will be texture
id mismatch between the tee branches.
Change-Id: I7261e1cd480cb8811c3c106bf7b9e2dd557fe48c
diff --git a/edgetpuvision/gst_native.py b/edgetpuvision/gst_native.py
index eb79089..c20e357 100644
--- a/edgetpuvision/gst_native.py
+++ b/edgetpuvision/gst_native.py
@@ -45,7 +45,7 @@
libgst.gst_context_writable_structure.argtypes = [ctypes.c_void_p]
libgst.gst_structure_set.restype = ctypes.c_void_p
libgst.gst_structure_set.argtypes = [ctypes.c_void_p, ctypes.c_char_p,
- ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
GST_MAP_INFO_POINTER = ctypes.POINTER(GstMapInfo)
libgst.gst_buffer_map.argtypes = [ctypes.c_void_p, GST_MAP_INFO_POINTER, ctypes.c_int]
libgst.gst_buffer_map.restype = ctypes.c_int
@@ -98,6 +98,13 @@
hash(GObject.TYPE_POINTER), wl_display, 0)
sink.set_context(context)
+def wrap_gl_context(gl_context):
+ context = Gst.Context.new('gst.gl.local_context', True)
+ structure = libgst.gst_context_writable_structure(hash(context))
+ libgst.gst_structure_set(structure, ctypes.c_char_p('context'.encode()),
+ hash(GObject.GType.from_name('GstGLContext')), hash(gl_context), 0)
+ return context
+
@contextlib.contextmanager
def _gst_buffer_map(buffer, flags):
ptr = hash(buffer)
diff --git a/edgetpuvision/gstreamer.py b/edgetpuvision/gstreamer.py
index 7df09d4..7557396 100644
--- a/edgetpuvision/gstreamer.py
+++ b/edgetpuvision/gstreamer.py
@@ -45,7 +45,7 @@
from PIL import Image
-from .gst_native import set_display_contexts
+from .gst_native import set_display_contexts, wrap_gl_context
from .pipelines import *
COMMAND_SAVE_FRAME = ' '
@@ -338,6 +338,20 @@
return Gst.PadProbeReturn.DROP
return Gst.PadProbeReturn.OK
+ # Our appsink needs to respond to CONTEXT queries so GL elements in that pipeline
+ # branch share GL context with glimagesink. The query isn't propagated from one
+ # tee brach to the others so it never reaches glimagesink.
+ def on_appsink_query(pad, info, glsink):
+ query = info.get_query()
+ if query.type == Gst.QueryType.CONTEXT:
+ gl_context = glsink.get_property('context')
+ _, context_type = query.parse_context_type()
+ if gl_context and context_type == 'gst.gl.local_context':
+ context = wrap_gl_context(gl_context)
+ query.set_context(context)
+ return Gst.PadProbeReturn.HANDLED
+ return Gst.PadProbeReturn.OK
+
window = Gtk.Window(Gtk.WindowType.TOPLEVEL)
window.set_title(WINDOW_TITLE)
window.set_default_size(layout.render_size.width, layout.render_size.height)
@@ -356,6 +370,10 @@
window.connect('delete-event', Gtk.main_quit)
window.show_all()
+ appsink = pipeline.get_by_name('appsink')
+ appsink.get_static_pad('sink').add_probe(Gst.PadProbeType.QUERY_DOWNSTREAM,
+ on_appsink_query, glsink)
+
with Worker(save_frame) as images, Commands() as get_command:
signals = {'appsink':
{'new-sample': functools.partial(on_new_sample,
diff --git a/edgetpuvision/pipelines.py b/edgetpuvision/pipelines.py
index 3a59d93..516cbc2 100644
--- a/edgetpuvision/pipelines.py
+++ b/edgetpuvision/pipelines.py
@@ -90,7 +90,6 @@
Filter('glupload'),
Tee(name='t')],
[Pad('t'),
- Filter('glupload'),
Queue(),
Pad('mixer')],
[Source('overlay', name='overlay'),
@@ -111,7 +110,6 @@
Filter('glupload'),
Tee(name='t')],
[Pad('t'),
- Filter('glupload'),
Queue(),
Pad('mixer')],
[Source('overlay', name='overlay'),
