| /* |
| * GStreamer |
| * Copyright (C) 2015 Lubosz Sarnecki <lubosz.sarnecki@collabora.co.uk> |
| * |
| * This library is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU Library General Public |
| * License as published by the Free Software Foundation; either |
| * version 2 of the License, or (at your option) any later version. |
| * |
| * This library is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| * Library General Public License for more details. |
| * |
| * You should have received a copy of the GNU Library General Public |
| * License along with this library; if not, write to the |
| * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, |
| * Boston, MA 02110-1301, USA. |
| */ |
| |
| #ifdef HAVE_CONFIG_H |
| #include "config.h" |
| #endif |
| |
| #include <stdio.h> |
| |
| #include "gl.h" |
| #include "gstgloverlaycompositor.h" |
| |
| GST_DEBUG_CATEGORY_STATIC (gst_gl_overlay_compositor_debug); |
| #define GST_CAT_DEFAULT gst_gl_overlay_compositor_debug |
| |
| /***************************************************************************** |
| * GstGLCompositionOverlay object is internally used by GstGLOverlayCompositor |
| *****************************************************************************/ |
| |
| #define GST_TYPE_GL_COMPOSITION_OVERLAY (gst_gl_composition_overlay_get_type()) |
| #define GST_GL_COMPOSITION_OVERLAY(obj) \ |
| (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_COMPOSITION_OVERLAY,\ |
| GstGLCompositionOverlay)) |
| |
| typedef struct _GstGLCompositionOverlay GstGLCompositionOverlay; |
| typedef struct _GstGLCompositionOverlayClass GstGLCompositionOverlayClass; |
| |
| static GType gst_gl_composition_overlay_get_type (void); |
| |
| /* *INDENT-OFF* */ |
| const gchar *fragment_shader = |
| "#ifdef GL_ES\n" |
| "precision mediump float;\n" |
| "#endif\n" |
| "varying vec2 v_texcoord;\n" |
| "uniform sampler2D tex;\n" |
| "void main(void)\n" |
| "{\n" |
| " vec4 t = texture2D(tex, v_texcoord);\n" |
| #if G_BYTE_ORDER == G_LITTLE_ENDIAN |
| " gl_FragColor = t.bgra;\n" |
| #else |
| " gl_FragColor = t.gbar;\n" |
| #endif |
| "}"; |
| /* *INDENT-ON* */ |
| |
| struct _GstGLCompositionOverlay |
| { |
| GstObject parent; |
| GstGLContext *context; |
| |
| GLuint vao; |
| GLuint index_buffer; |
| GLuint position_buffer; |
| GLuint texcoord_buffer; |
| GLint position_attrib; |
| GLint texcoord_attrib; |
| |
| GLfloat positions[16]; |
| |
| GLuint texture_id; |
| GstGLMemory *gl_memory; |
| GstVideoOverlayRectangle *rectangle; |
| }; |
| |
| struct _GstGLCompositionOverlayClass |
| { |
| GstObjectClass object_class; |
| }; |
| |
| G_DEFINE_TYPE (GstGLCompositionOverlay, gst_gl_composition_overlay, |
| GST_TYPE_OBJECT); |
| |
| static void |
| gst_gl_composition_overlay_init_vertex_buffer (GstGLContext * context, |
| gpointer overlay_pointer) |
| { |
| const GstGLFuncs *gl = context->gl_vtable; |
| GstGLCompositionOverlay *overlay = |
| (GstGLCompositionOverlay *) overlay_pointer; |
| |
| /* *INDENT-OFF* */ |
| static const GLfloat texcoords[] = { |
| 1.0f, 0.0f, |
| 0.0f, 0.0f, |
| 0.0f, 1.0f, |
| 1.0f, 1.0f |
| }; |
| |
| static const GLushort indices[] = { |
| 0, 1, 2, 0, 2, 3 |
| }; |
| /* *INDENT-ON* */ |
| |
| if (gl->GenVertexArrays) { |
| gl->GenVertexArrays (1, &overlay->vao); |
| gl->BindVertexArray (overlay->vao); |
| } |
| |
| gl->GenBuffers (1, &overlay->position_buffer); |
| gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer); |
| gl->BufferData (GL_ARRAY_BUFFER, 4 * 4 * sizeof (GLfloat), overlay->positions, |
| GL_STATIC_DRAW); |
| |
| /* Load the vertex position */ |
| gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE, |
| 4 * sizeof (GLfloat), NULL); |
| |
| gl->GenBuffers (1, &overlay->texcoord_buffer); |
| gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer); |
| gl->BufferData (GL_ARRAY_BUFFER, 4 * 2 * sizeof (GLfloat), texcoords, |
| GL_STATIC_DRAW); |
| |
| /* Load the texture coordinate */ |
| gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE, |
| 2 * sizeof (GLfloat), NULL); |
| |
| gl->GenBuffers (1, &overlay->index_buffer); |
| gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer); |
| gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, |
| GL_STATIC_DRAW); |
| |
| gl->EnableVertexAttribArray (overlay->position_attrib); |
| gl->EnableVertexAttribArray (overlay->texcoord_attrib); |
| |
| if (gl->GenVertexArrays) { |
| gl->BindVertexArray (0); |
| } |
| |
| gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); |
| gl->BindBuffer (GL_ARRAY_BUFFER, 0); |
| } |
| |
| static void |
| gst_gl_composition_overlay_free_vertex_buffer (GstGLContext * context, |
| gpointer overlay_pointer) |
| { |
| const GstGLFuncs *gl = context->gl_vtable; |
| GstGLCompositionOverlay *overlay = |
| (GstGLCompositionOverlay *) overlay_pointer; |
| if (overlay->vao) { |
| gl->DeleteVertexArrays (1, &overlay->vao); |
| overlay->vao = 0; |
| } |
| |
| if (overlay->position_buffer) { |
| gl->DeleteBuffers (1, &overlay->position_buffer); |
| overlay->position_buffer = 0; |
| } |
| |
| if (overlay->texcoord_buffer) { |
| gl->DeleteBuffers (1, &overlay->position_buffer); |
| overlay->position_buffer = 0; |
| } |
| |
| if (overlay->index_buffer) { |
| gl->DeleteBuffers (1, &overlay->index_buffer); |
| overlay->index_buffer = 0; |
| } |
| } |
| |
| static void |
| gst_gl_composition_overlay_bind_vertex_buffer (GstGLCompositionOverlay * |
| overlay) |
| { |
| const GstGLFuncs *gl = overlay->context->gl_vtable; |
| gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer); |
| gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE, |
| 4 * sizeof (GLfloat), NULL); |
| |
| gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer); |
| gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE, |
| 2 * sizeof (GLfloat), NULL); |
| |
| gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer); |
| |
| gl->EnableVertexAttribArray (overlay->position_attrib); |
| gl->EnableVertexAttribArray (overlay->texcoord_attrib); |
| } |
| |
| static void |
| gst_gl_composition_overlay_finalize (GObject * object) |
| { |
| GstGLCompositionOverlay *overlay; |
| |
| overlay = GST_GL_COMPOSITION_OVERLAY (object); |
| |
| if (overlay->gl_memory) |
| gst_memory_unref ((GstMemory *) overlay->gl_memory); |
| |
| if (overlay->context) { |
| gst_gl_context_thread_add (overlay->context, |
| gst_gl_composition_overlay_free_vertex_buffer, overlay); |
| gst_object_unref (overlay->context); |
| } |
| |
| G_OBJECT_CLASS (gst_gl_composition_overlay_parent_class)->finalize (object); |
| } |
| |
| |
| static void |
| gst_gl_composition_overlay_class_init (GstGLCompositionOverlayClass * klass) |
| { |
| G_OBJECT_CLASS (klass)->finalize = gst_gl_composition_overlay_finalize; |
| } |
| |
| static void |
| gst_gl_composition_overlay_init (GstGLCompositionOverlay * overlay) |
| { |
| } |
| |
| static void |
| gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay * |
| overlay, GstBuffer * video_buffer) |
| { |
| gint comp_x, comp_y; |
| guint comp_width, comp_height; |
| GstVideoMeta *meta; |
| guint width, height; |
| |
| float rel_x, rel_y, rel_w, rel_h; |
| |
| meta = gst_buffer_get_video_meta (video_buffer); |
| |
| gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle, |
| &comp_x, &comp_y, &comp_width, &comp_height); |
| |
| width = meta->width; |
| height = meta->height; |
| |
| /* calculate relative position */ |
| rel_x = (float) comp_x / (float) width; |
| rel_y = (float) comp_y / (float) height; |
| |
| rel_w = (float) comp_width / (float) width; |
| rel_h = (float) comp_height / (float) height; |
| |
| /* transform from [0,1] to [-1,1], invert y axis */ |
| rel_x = rel_x * 2.0 - 1.0; |
| rel_y = (1.0 - rel_y) * 2.0 - 1.0; |
| rel_w = rel_w * 2.0; |
| rel_h = rel_h * 2.0; |
| |
| /* initialize position array */ |
| overlay->positions[0] = rel_x + rel_w; |
| overlay->positions[1] = rel_y; |
| overlay->positions[2] = 0.0; |
| overlay->positions[3] = 1.0; |
| overlay->positions[4] = rel_x; |
| overlay->positions[5] = rel_y; |
| overlay->positions[6] = 0.0; |
| overlay->positions[7] = 1.0; |
| overlay->positions[8] = rel_x; |
| overlay->positions[9] = rel_y - rel_h; |
| overlay->positions[10] = 0.0; |
| overlay->positions[11] = 1.0; |
| overlay->positions[12] = rel_x + rel_w; |
| overlay->positions[13] = rel_y - rel_h; |
| overlay->positions[14] = 0.0; |
| overlay->positions[15] = 1.0; |
| |
| gst_gl_context_thread_add (overlay->context, |
| gst_gl_composition_overlay_free_vertex_buffer, overlay); |
| |
| gst_gl_context_thread_add (overlay->context, |
| gst_gl_composition_overlay_init_vertex_buffer, overlay); |
| |
| GST_DEBUG |
| ("overlay position: (%d,%d) size: %dx%d video size: %dx%d", |
| comp_x, comp_y, comp_width, comp_height, meta->width, meta->height); |
| } |
| |
| /* helper object API functions */ |
| |
| static GstGLCompositionOverlay * |
| gst_gl_composition_overlay_new (GstGLContext * context, |
| GstVideoOverlayRectangle * rectangle, |
| GLint position_attrib, GLint texcoord_attrib) |
| { |
| GstGLCompositionOverlay *overlay = |
| g_object_new (GST_TYPE_GL_COMPOSITION_OVERLAY, NULL); |
| |
| overlay->gl_memory = NULL; |
| overlay->texture_id = -1; |
| overlay->rectangle = rectangle; |
| overlay->context = gst_object_ref (context); |
| overlay->vao = 0; |
| overlay->position_attrib = position_attrib; |
| overlay->texcoord_attrib = texcoord_attrib; |
| |
| GST_DEBUG_OBJECT (overlay, "Created new GstGLCompositionOverlay"); |
| |
| return overlay; |
| } |
| |
| static void |
| _video_frame_unmap_and_free (gpointer user_data) |
| { |
| GstVideoFrame *frame = user_data; |
| |
| gst_video_frame_unmap (frame); |
| g_slice_free (GstVideoFrame, frame); |
| } |
| |
| static void |
| gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay, |
| GstBuffer * buf) |
| { |
| GstGLMemory *comp_gl_memory = NULL; |
| GstBuffer *comp_buffer = NULL; |
| GstBuffer *overlay_buffer = NULL; |
| GstVideoInfo vinfo; |
| GstVideoMeta *vmeta; |
| GstVideoFrame *comp_frame; |
| GstVideoFrame gl_frame; |
| |
| comp_buffer = |
| gst_video_overlay_rectangle_get_pixels_unscaled_argb (overlay->rectangle, |
| GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA); |
| |
| comp_frame = g_slice_new (GstVideoFrame); |
| |
| vmeta = gst_buffer_get_video_meta (comp_buffer); |
| gst_video_info_set_format (&vinfo, vmeta->format, vmeta->width, |
| vmeta->height); |
| vinfo.stride[0] = vmeta->stride[0]; |
| |
| if (gst_video_frame_map (comp_frame, &vinfo, comp_buffer, GST_MAP_READ)) { |
| gst_gl_composition_overlay_add_transformation (overlay, buf); |
| |
| comp_gl_memory = |
| gst_gl_memory_wrapped (overlay->context, &comp_frame->info, 0, NULL, |
| comp_frame->data[0], comp_frame, _video_frame_unmap_and_free); |
| |
| overlay_buffer = gst_buffer_new (); |
| gst_buffer_append_memory (overlay_buffer, (GstMemory *) comp_gl_memory); |
| |
| if (!gst_video_frame_map (&gl_frame, &comp_frame->info, overlay_buffer, |
| GST_MAP_READ | GST_MAP_GL)) { |
| gst_buffer_unref (overlay_buffer); |
| _video_frame_unmap_and_free (comp_frame); |
| GST_WARNING_OBJECT (overlay, "Cannot upload overlay texture"); |
| return; |
| } |
| |
| gst_memory_ref ((GstMemory *) comp_gl_memory); |
| overlay->gl_memory = comp_gl_memory; |
| overlay->texture_id = comp_gl_memory->tex_id; |
| |
| gst_buffer_unref (overlay_buffer); |
| gst_video_frame_unmap (&gl_frame); |
| |
| GST_DEBUG ("uploaded overlay texture %d", overlay->texture_id); |
| } else { |
| g_slice_free (GstVideoFrame, comp_frame); |
| } |
| } |
| |
| static void |
| gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay, |
| GstGLShader * shader) |
| { |
| const GstGLFuncs *gl = overlay->context->gl_vtable; |
| if (gl->GenVertexArrays) |
| gl->BindVertexArray (overlay->vao); |
| else |
| gst_gl_composition_overlay_bind_vertex_buffer (overlay); |
| |
| if (overlay->texture_id != -1) |
| gl->BindTexture (GL_TEXTURE_2D, overlay->texture_id); |
| gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); |
| } |
| |
| |
| /******************************************************************** |
| * GstGLOverlayCompositor object, the public helper object to render |
| * GstVideoCompositionOverlayMeta |
| ********************************************************************/ |
| |
| #define DEBUG_INIT \ |
| GST_DEBUG_CATEGORY_INIT (gst_gl_overlay_compositor_debug, \ |
| "gloverlaycompositor", 0, "overlaycompositor"); |
| |
| G_DEFINE_TYPE_WITH_CODE (GstGLOverlayCompositor, gst_gl_overlay_compositor, |
| GST_TYPE_OBJECT, DEBUG_INIT); |
| |
| static void gst_gl_overlay_compositor_finalize (GObject * object); |
| static gboolean _is_rectangle_in_overlays (GList * overlays, |
| GstVideoOverlayRectangle * rectangle); |
| static gboolean _is_overlay_in_rectangles (GstVideoOverlayComposition * |
| composition, GstGLCompositionOverlay * overlay); |
| |
| static void |
| gst_gl_overlay_compositor_class_init (GstGLOverlayCompositorClass * klass) |
| { |
| G_OBJECT_CLASS (klass)->finalize = gst_gl_overlay_compositor_finalize; |
| } |
| |
| static void |
| gst_gl_overlay_compositor_init (GstGLOverlayCompositor * compositor) |
| { |
| } |
| |
| static void |
| gst_gl_overlay_compositor_init_gl (GstGLContext * context, |
| gpointer compositor_pointer) |
| { |
| GstGLOverlayCompositor *compositor = |
| (GstGLOverlayCompositor *) compositor_pointer; |
| |
| if (!gst_gl_shader_compile_with_default_v_and_check (compositor->shader, |
| fragment_shader, &compositor->position_attrib, |
| &compositor->texcoord_attrib)) { |
| GST_ERROR ("could not initialize shader."); |
| } |
| } |
| |
| GstGLOverlayCompositor * |
| gst_gl_overlay_compositor_new (GstGLContext * context) |
| { |
| GstGLOverlayCompositor *compositor = |
| g_object_new (GST_TYPE_GL_OVERLAY_COMPOSITOR, NULL); |
| |
| compositor->context = gst_object_ref (context); |
| |
| compositor->shader = gst_gl_shader_new (compositor->context); |
| |
| gst_gl_context_thread_add (compositor->context, |
| gst_gl_overlay_compositor_init_gl, compositor); |
| |
| GST_DEBUG_OBJECT (compositor, "Created new GstGLOverlayCompositor"); |
| |
| return compositor; |
| } |
| |
| static void |
| gst_gl_overlay_compositor_finalize (GObject * object) |
| { |
| GstGLOverlayCompositor *compositor; |
| |
| compositor = GST_GL_OVERLAY_COMPOSITOR (object); |
| |
| gst_gl_overlay_compositor_free_overlays (compositor); |
| |
| if (compositor->context) |
| gst_object_unref (compositor->context); |
| |
| if (compositor->shader) { |
| gst_object_unref (compositor->shader); |
| compositor->shader = NULL; |
| } |
| |
| G_OBJECT_CLASS (gst_gl_overlay_compositor_parent_class)->finalize (object); |
| } |
| |
| static gboolean |
| _is_rectangle_in_overlays (GList * overlays, |
| GstVideoOverlayRectangle * rectangle) |
| { |
| GList *l; |
| |
| for (l = overlays; l != NULL; l = l->next) { |
| GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data; |
| if (overlay->rectangle == rectangle) |
| return TRUE; |
| } |
| return FALSE; |
| } |
| |
| static gboolean |
| _is_overlay_in_rectangles (GstVideoOverlayComposition * composition, |
| GstGLCompositionOverlay * overlay) |
| { |
| guint i; |
| |
| for (i = 0; i < gst_video_overlay_composition_n_rectangles (composition); i++) { |
| GstVideoOverlayRectangle *rectangle = |
| gst_video_overlay_composition_get_rectangle (composition, i); |
| if (overlay->rectangle == rectangle) |
| return TRUE; |
| } |
| return FALSE; |
| } |
| |
| |
| void |
| gst_gl_overlay_compositor_free_overlays (GstGLOverlayCompositor * compositor) |
| { |
| GList *l = compositor->overlays; |
| while (l != NULL) { |
| GList *next = l->next; |
| GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data; |
| compositor->overlays = g_list_delete_link (compositor->overlays, l); |
| gst_object_unref (overlay); |
| l = next; |
| } |
| g_list_free (compositor->overlays); |
| compositor->overlays = NULL; |
| } |
| |
| void |
| gst_gl_overlay_compositor_upload_overlays (GstGLOverlayCompositor * compositor, |
| GstBuffer * buf) |
| { |
| GstVideoOverlayCompositionMeta *composition_meta; |
| |
| composition_meta = gst_buffer_get_video_overlay_composition_meta (buf); |
| if (composition_meta) { |
| GstVideoOverlayComposition *composition = NULL; |
| guint num_overlays, i; |
| GList *l = compositor->overlays; |
| |
| GST_DEBUG ("GstVideoOverlayCompositionMeta found."); |
| |
| composition = composition_meta->overlay; |
| num_overlays = gst_video_overlay_composition_n_rectangles (composition); |
| |
| /* add new overlays to list */ |
| for (i = 0; i < num_overlays; i++) { |
| GstVideoOverlayRectangle *rectangle = |
| gst_video_overlay_composition_get_rectangle (composition, i); |
| |
| if (!_is_rectangle_in_overlays (compositor->overlays, rectangle)) { |
| GstGLCompositionOverlay *overlay = |
| gst_gl_composition_overlay_new (compositor->context, rectangle, |
| compositor->position_attrib, compositor->texcoord_attrib); |
| |
| gst_gl_composition_overlay_upload (overlay, buf); |
| |
| compositor->overlays = g_list_append (compositor->overlays, overlay); |
| } |
| } |
| |
| /* remove old overlays from list */ |
| while (l != NULL) { |
| GList *next = l->next; |
| GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data; |
| if (!_is_overlay_in_rectangles (composition, overlay)) { |
| compositor->overlays = g_list_delete_link (compositor->overlays, l); |
| gst_object_unref (overlay); |
| } |
| l = next; |
| } |
| } else { |
| gst_gl_overlay_compositor_free_overlays (compositor); |
| } |
| } |
| |
| void |
| gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor) |
| { |
| const GstGLFuncs *gl = compositor->context->gl_vtable; |
| if (compositor->overlays != NULL) { |
| GList *l; |
| |
| gl->Enable (GL_BLEND); |
| gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); |
| |
| gst_gl_shader_use (compositor->shader); |
| gl->ActiveTexture (GL_TEXTURE0); |
| gst_gl_shader_set_uniform_1i (compositor->shader, "tex", 0); |
| |
| for (l = compositor->overlays; l != NULL; l = l->next) { |
| GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data; |
| gst_gl_composition_overlay_draw (overlay, compositor->shader); |
| } |
| |
| gl->BindTexture (GL_TEXTURE_2D, 0); |
| gl->Disable (GL_BLEND); |
| } |
| } |
| |
| GstCaps * |
| gst_gl_overlay_compositor_add_caps (GstCaps * caps) |
| { |
| GstCaps *composition_caps; |
| int i; |
| |
| composition_caps = gst_caps_copy (caps); |
| |
| for (i = 0; i < gst_caps_get_size (composition_caps); i++) { |
| GstCapsFeatures *f = gst_caps_get_features (composition_caps, i); |
| gst_caps_features_add (f, |
| GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); |
| } |
| |
| caps = gst_caps_merge (composition_caps, caps); |
| |
| return caps; |
| } |