Render full size overlays, draw with glvideomixer With this approach there's no need for downscaling as video and overlay rendering are decoupled in glvideomixer, so we can do 1080p@30 in real time. Video frames are displayed at their normal rate and overlays are rendered at a potentially slower rate if we can't keep up without the video stalling. The main improvment here is a custom source element that enables us to hit a fast GL texture upload path where we draw the overlay with cairo/rsvg straight into a dma-buf (allocated by ion) and use that as a texture in glvideomixer. Change-Id: Icf69e29ab55a95122754a6e9f2c63ac7adc2d465
diff --git a/edgetpuvision/apps.py b/edgetpuvision/apps.py index 9c5febd..e84df8a 100644 --- a/edgetpuvision/apps.py +++ b/edgetpuvision/apps.py
@@ -43,8 +43,6 @@ parser.add_argument('--source', help='/dev/videoN:FMT:WxH:N/D or .mp4 file or image file', default='/dev/video0:YUY2:1280x720:30/1') - parser.add_argument('--downscale', type=float, default=2.0, - help='Downscale factor for .mp4 file rendering') parser.add_argument('--loop', default=False, action='store_true', help='Loop input video file') parser.add_argument('--displaymode', type=Display, choices=Display, default=Display.FULLSCREEN, @@ -54,7 +52,6 @@ if not run_gen(render_gen(args), source=args.source, - downscale=args.downscale, loop=args.loop, display=args.displaymode): print('Invalid source argument:', args.source)
diff --git a/edgetpuvision/gst_native.py b/edgetpuvision/gst_native.py index 5a0c562..acce316 100644 --- a/edgetpuvision/gst_native.py +++ b/edgetpuvision/gst_native.py
@@ -1,6 +1,7 @@ import cairo import contextlib import ctypes +import threading import gi gi.require_version('Gdk', '3.0') @@ -53,16 +54,12 @@ libcairo.cairo_surface_flush.argtypes = [ctypes.c_void_p] libcairo.cairo_surface_destroy.restype = None libcairo.cairo_surface_destroy.argtypes = [ctypes.c_void_p] -libcairo.cairo_surface_status.restype = ctypes.c_int -libcairo.cairo_surface_status.argtypes = [ctypes.c_void_p] libcairo.cairo_format_stride_for_width.restype = ctypes.c_int libcairo.cairo_format_stride_for_width.argtypes = [ctypes.c_int, ctypes.c_int] libcairo.cairo_create.restype = ctypes.c_void_p libcairo.cairo_create.argtypes = [ctypes.c_void_p] libcairo.cairo_destroy.restype = None libcairo.cairo_destroy.argtypes = [ctypes.c_void_p] -libcairo.cairo_scale.restype = None -libcairo.cairo_scale.argtypes = [ctypes.c_void_p, ctypes.c_double, ctypes.c_double] librsvg = ctypes.CDLL('librsvg-2.so.2') librsvg.rsvg_handle_new_from_data.restype = ctypes.c_void_p @@ -102,22 +99,19 @@ finally: libgst.gst_buffer_unmap(ptr, mapping) -# GStreamer Element that attaches VideoOverlayComposition to buffers passing by. -class OverlayInjector(GstBase.BaseTransform): +class OverlaySource(GstBase.BaseSrc): __gstmetadata__ = ('<longname>', '<class>', '<description>', '<author>') __gsttemplates__ = (Gst.PadTemplate.new('src', Gst.PadDirection.SRC, Gst.PadPresence.ALWAYS, - Gst.Caps.new_any()), - Gst.PadTemplate.new('sink', - Gst.PadDirection.SINK, - Gst.PadPresence.ALWAYS, - Gst.Caps.new_any())) + Gst.Caps.from_string( + 'video/x-raw,format=BGRA,framerate=0/1' + ))) @staticmethod def _plugin_init(plugin): - gtype = GObject.type_register(OverlayInjector) - Gst.Element.register(plugin, 'overlayinjector', 0, gtype) + gtype = GObject.type_register(OverlaySource) + Gst.Element.register(plugin, 'overlaysrc', 0, gtype) return True @staticmethod @@ -127,7 +121,7 @@ version[0], version[1], # GStreamer version '', # name '', # description - OverlayInjector._plugin_init, # init_func + OverlaySource._plugin_init, # init_func '', # version 'unknown', # license '', # source @@ -136,60 +130,135 @@ ) def __init__(self): - GstBase.BaseTransform.__init__(self) - GstBase.BaseTransform.set_in_place(self, True) - self.render_size = None + GstBase.BaseSrc.__init__(self) + self.set_format(Gst.Format.TIME) + self.set_do_timestamp(False) + self.set_live(True) + self.cond = threading.Condition() + self.width = 0 + self.height = 0 + self.flushing = False + self.eos = False self.svg = None - self.rendered_svg = None - self.composition = None - self.scale_factor = 0.75 - - def set_svg(self, svg, render_size): - self.svg = svg - self.render_size = render_size - - def do_transform_ip(self, frame_buf): - self.render() - if self.composition: - # Note: Buffer IS writable (ref is 1 in native land). However gst-python - # took an additional ref so it's now 2 and gst_buffer_is_writable - # returns false. We can't modify the buffer without fiddling with refcount. - if frame_buf.mini_object.refcount != 2: - return Gst.FlowReturn.ERROR - frame_buf.mini_object.refcount -= 1 - GstVideo.buffer_add_video_overlay_composition_meta(frame_buf, self.composition) - frame_buf.mini_object.refcount += 1 - return Gst.FlowReturn.OK + self.pts = 0 - def render(self): - if not self.svg: - self.composition = None - self.rendered_svg = None - return + def do_decide_allocation(self, query): + if query.get_n_allocation_pools() > 0: + pool, size, min_buffers, max_buffers = query.parse_nth_allocation_pool(0) + query.set_nth_allocation_pool(0, pool, size, min_buffers, min(max_buffers, 3)) + return GstBase.BaseSrc.do_decide_allocation(self, query) - if self.svg == self.rendered_svg: - return + def do_event(self, event): + if event.type == Gst.EventType.SEEK: + _, _, flags, _, _, _, _ = event.parse_seek() + if flags | Gst.SeekFlags.FLUSH: + self.send_event(Gst.Event.new_flush_start()) + self.send_event(Gst.Event.new_flush_stop(True)) + return True + return GstBase.BaseSrc.do_event(self, event) - overlay_size = self.render_size * self.scale_factor - stride = libcairo.cairo_format_stride_for_width( - int(cairo.FORMAT_ARGB32), overlay_size.width) - overlay_buffer = Gst.Buffer.new_allocate(None, - stride * overlay_size.height) - with _gst_buffer_map(overlay_buffer, Gst.MapFlags.WRITE) as mapped: - # Fill with transparency and create surface from buffer. + def set_eos(self): + with self.cond: + self.eos = True + + def do_start (self): + self.set_svg(None, 0) + return True + + def do_stop (self): + self.set_svg(None, 0) + return True + + def set_svg(self, svg, pts): + with self.cond: + self.svg = svg + self.pts = pts + self.eos = False + self.cond.notify_all() + + def set_flushing(self, flushing): + with self.cond: + self.flushing = flushing + self.cond.notify_all() + + def do_set_caps(self, caps): + structure = caps.get_structure(0) + self.width = structure.get_value('width') + self.height = structure.get_value('height') + return True + + def do_unlock(self): + self.set_flushing(True) + return True + + def do_unlock_stop(self): + self.set_flushing(False) + return True + + def get_flow_return_locked(self, default=None): + if self.eos: + self.eos = False + self.svg = None + return Gst.FlowReturn.EOS + if self.flushing: + return Gst.FlowReturn.FLUSHING + return default + + def do_fill(self, offset, size, buf): + with self.cond: + result = self.get_flow_return_locked() + if result: + return result + + while self.svg is None: + self.cond.wait() + result = self.get_flow_return_locked() + if result: + return result + + assert self.svg is not None + svg = self.svg + pts = self.pts + self.svg = None + + # Note: Buffer IS writable (ref is 1 in native land). However gst-python + # took an additional ref so it's now 2 and gst_buffer_is_writable + # returns false. We can't modify the buffer without fiddling with refcount. + assert buf.mini_object.refcount == 2 + buf.mini_object.refcount = 1 + try: + self.render_svg(svg, buf) + buf.pts = pts + finally: + buf.mini_object.refcount = 2 + + with self.cond: + return self.get_flow_return_locked(Gst.FlowReturn.OK) + + def render_svg(self, svg, buf): + with _gst_buffer_map(buf, Gst.MapFlags.WRITE) as mapped: + stride = libcairo.cairo_format_stride_for_width( + int(cairo.FORMAT_ARGB32), self.width) + assert len(mapped) >= stride * self.height + + # Fill with transparency. ctypes.memset(ctypes.addressof(mapped), 0, ctypes.sizeof(mapped)) + + # If svg is '' (can't be None here) we return 100% transparency. + if not svg: + return + surface = libcairo.cairo_image_surface_create_for_data( ctypes.addressof(mapped), int(cairo.FORMAT_ARGB32), - overlay_size.width, - overlay_size.height, + self.width, + self.height, stride) # Render the SVG overlay. - data = self.svg.encode('utf-8') + data = svg.encode('utf-8') context = libcairo.cairo_create(surface) - libcairo.cairo_scale(context, self.scale_factor, self.scale_factor) handle = librsvg.rsvg_handle_new_from_data(data, len(data), 0) librsvg.rsvg_handle_render_cairo(handle, context) librsvg.rsvg_handle_close(handle, 0) @@ -198,14 +267,4 @@ libcairo.cairo_surface_destroy(surface) libcairo.cairo_destroy(context) - # Attach overlay to VideoOverlayComposition. - GstVideo.buffer_add_video_meta(overlay_buffer, - GstVideo.VideoFrameFlags.NONE, GstVideo.VideoFormat.BGRA, - overlay_size.width, overlay_size.height) - rect = GstVideo.VideoOverlayRectangle.new_raw(overlay_buffer, - 0, 0, self.render_size.width, self.render_size.height, - GstVideo.VideoOverlayFormatFlags.PREMULTIPLIED_ALPHA) - self.composition = GstVideo.VideoOverlayComposition.new(rect) - self.rendered_svg = self.svg - -OverlayInjector.plugin_register() +OverlaySource.plugin_register()
diff --git a/edgetpuvision/gstreamer.py b/edgetpuvision/gstreamer.py index 6c0018f..11d2560 100644 --- a/edgetpuvision/gstreamer.py +++ b/edgetpuvision/gstreamer.py
@@ -145,12 +145,15 @@ assert len(streams) == 1 return streams[0] -def is_seekable(element): +def get_seek_element(pipeline): + element = pipeline.get_by_name('glsink') + if not element: + element = pipeline query = Gst.Query.new_seeking(Gst.Format.TIME) if element.query(query): _, seekable, _, _ = query.parse_seeking() - return seekable - return False + return element + return None @contextlib.contextmanager def pull_sample(sink): @@ -159,21 +162,22 @@ result, mapinfo = buf.map(Gst.MapFlags.READ) if result: - yield sample, mapinfo.data + yield sample, mapinfo.data, buf.pts buf.unmap(mapinfo) def new_sample_callback(process): def callback(sink, pipeline): - with pull_sample(sink) as (sample, data): + with pull_sample(sink) as (sample, data, pts): process(data, caps_size(sample.get_caps())) return Gst.FlowReturn.OK return callback def on_bus_message(bus, message, pipeline, loop): if message.type == Gst.MessageType.EOS: - if loop and is_seekable(pipeline): + seek_element = get_seek_element(pipeline) + if loop and seek_element: flags = Gst.SeekFlags.FLUSH | Gst.SeekFlags.KEY_UNIT - if not pipeline.seek_simple(Gst.Format.TIME, flags, 0): + if not seek_element.seek_simple(Gst.Format.TIME, flags, 0): Gtk.main_quit() else: Gtk.main_quit() @@ -185,8 +189,13 @@ sys.stderr.write('Error: %s: %s\n' % (err, debug)) Gtk.main_quit() +def on_sink_eos(sink, pipeline): + overlay = pipeline.get_by_name('overlay') + if overlay: + overlay.set_eos() + def on_new_sample(sink, pipeline, render_overlay, layout, images, get_command): - with pull_sample(sink) as (sample, data): + with pull_sample(sink) as (sample, data, pts): custom_command = None save_frame = False @@ -206,25 +215,25 @@ command=custom_command) overlay = pipeline.get_by_name('overlay') if overlay: - overlay.set_svg(svg, layout.render_size) + overlay.set_svg(svg, pts) if save_frame: images.put((data, layout.inference_size, svg)) return Gst.FlowReturn.OK -def run_gen(render_overlay_gen, *, source, downscale, loop, display): +def run_gen(render_overlay_gen, *, source, loop, display): inference_size = render_overlay_gen.send(None) # Initialize. + next(render_overlay_gen) return run(inference_size, lambda tensor, layout, command: render_overlay_gen.send((tensor, layout, command)), source=source, - downscale=downscale, loop=loop, display=display) -def run(inference_size, render_overlay, *, source, downscale, loop, display): - result = get_pipeline(source, inference_size, downscale, display) +def run(inference_size, render_overlay, *, source, loop, display): + result = get_pipeline(source, inference_size, display) if result: layout, pipeline = result run_pipeline(pipeline, layout, loop, render_overlay, display) @@ -232,7 +241,7 @@ return False -def get_pipeline(source, inference_size, downscale, display): +def get_pipeline(source, inference_size, display): fmt = parse_format(source) if fmt: layout = make_layout(inference_size, fmt.size) @@ -241,7 +250,7 @@ filename = os.path.expanduser(source) if os.path.isfile(filename): info = get_video_info(filename) - render_size = Size(info.get_width(), info.get_height()) / downscale + render_size = caps_size(info.get_caps()) layout = make_layout(inference_size, render_size) return layout, file_pipline(info.is_image(), filename, layout, display) @@ -315,7 +324,8 @@ render_overlay=functools.partial(render_overlay, layout=layout), layout=layout, images=images, - get_command=get_command)}, + get_command=get_command), + 'eos' : on_sink_eos}, **(signals or {}) }
diff --git a/edgetpuvision/pipelines.py b/edgetpuvision/pipelines.py index d79bf2b..739d400 100644 --- a/edgetpuvision/pipelines.py +++ b/edgetpuvision/pipelines.py
@@ -45,6 +45,8 @@ # Display def image_display_pipeline(filename, layout): return ( + [Filter('glvideomixer', name='mixer', background='black'), + display_sink(sync=False)], [decoded_file_src(filename), Tee(name='t')], [Pad('t'), @@ -53,24 +55,35 @@ Filter('videoscale'), Caps('video/x-raw', format='RGBA', width=layout.render_size.width, height=layout.render_size.height), Filter('imagefreeze'), - Filter('overlayinjector', name='overlay'), - display_sink()], + Filter('glupload'), + Pad('mixer')], + [Source('overlay', name='overlay'), + Caps('video/x-raw', format='BGRA', width=layout.render_size.width, height=layout.render_size.height), + Filter('glupload'), + Queue(max_size_buffers=1), + Pad('mixer')], [Pad('t'), Queue(), inference_pipeline(layout, stillimage=True)], ) -def video_display_pipeline(filename, layout,): + +def video_display_pipeline(filename, layout): return ( + [Filter('glvideomixer', name='mixer', background='black'), + display_sink(sync=True)], [decoded_file_src(filename), Filter('glupload'), Tee(name='t')], [Pad('t'), + Filter('glupload'), + Queue(), + Pad('mixer')], + [Source('overlay', name='overlay'), + Caps('video/x-raw', format='BGRA', width=layout.render_size.width, height=layout.render_size.height), + Filter('glupload'), Queue(max_size_buffers=1), - Filter('glfilterbin', filter='glcolorscale'), - Filter('overlayinjector', name='overlay'), - Caps('video/x-raw', width=layout.render_size.width, height=layout.render_size.height), - display_sink()], + Pad('mixer')], [Pad('t'), Queue(max_size_buffers=1, leaky='downstream'), inference_pipeline(layout)], @@ -78,14 +91,20 @@ def camera_display_pipeline(fmt, layout): return ( + [Filter('glvideomixer', name='mixer', background='black'), + display_sink(sync=False)], [v4l2_src(fmt), Filter('glupload'), Tee(name='t')], - [Pad(name='t'), - Queue(max_size_buffers=1, leaky='downstream'), - Filter('glfilterbin', filter='glcolorscale'), - Filter('overlayinjector', name='overlay'), - display_sink()], + [Pad('t'), + Filter('glupload'), + Queue(), + Pad('mixer')], + [Source('overlay', name='overlay'), + Caps('video/x-raw', format='BGRA', width=layout.render_size.width, height=layout.render_size.height), + Filter('glupload'), + Queue(max_size_buffers=1), + Pad('mixer')], [Pad(name='t'), Queue(max_size_buffers=1, leaky='downstream'), inference_pipeline(layout)],