Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 1 | /* |
| 2 | * GStreamer |
| 3 | * Copyright (C) 2013 Miguel Casas-Sanchez <miguelecasassanchez@gmail.com> |
Luis de Bethencourt | 447c1b1 | 2015-12-12 20:07:32 +0000 | [diff] [blame] | 4 | * |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 5 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 6 | * copy of this software and associated documentation files (the "Software"), |
| 7 | * to deal in the Software without restriction, including without limitation |
| 8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 9 | * and/or sell copies of the Software, and to permit persons to whom the |
| 10 | * Software is furnished to do so, subject to the following conditions: |
| 11 | * |
| 12 | * The above copyright notice and this permission notice shall be included in |
| 13 | * all copies or substantial portions of the Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| 21 | * DEALINGS IN THE SOFTWARE. |
| 22 | * |
| 23 | * Alternatively, the contents of this file may be used under the |
| 24 | * GNU Lesser General Public License Version 2.1 (the "LGPL"), in |
| 25 | * which case the following provisions apply instead of the ones |
| 26 | * mentioned above: |
| 27 | * |
| 28 | * This library is free software; you can redistribute it and/or |
| 29 | * modify it under the terms of the GNU Library General Public |
| 30 | * License as published by the Free Software Foundation; either |
| 31 | * version 2 of the License, or (at your option) any later version. |
| 32 | * |
| 33 | * This library is distributed in the hope that it will be useful, |
| 34 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 35 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 36 | * Library General Public License for more details. |
| 37 | * |
| 38 | * You should have received a copy of the GNU Library General Public |
| 39 | * License along with this library; if not, write to the |
| 40 | * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, |
| 41 | * Boston, MA 02110-1301, USA. |
| 42 | */ |
| 43 | |
| 44 | /** |
| 45 | * SECTION:element-grabcut |
| 46 | * |
| 47 | * |
| 48 | * This element is a wrapper around OpenCV grabcut implementation. GrabCut is an |
| 49 | * image segmentation method based on graph cuts technique. It can be seen as a |
| 50 | * way of fine-grain segmenting the image from some FG and BG "seed" areas. The |
Luis de Bethencourt | 447c1b1 | 2015-12-12 20:07:32 +0000 | [diff] [blame] | 51 | * OpenCV implementation follows the article [1]. |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 52 | * The "seed" areas are taken in this element from either an input bounding box |
| 53 | * coming from a face detection, or from alpha channel values. The input box is |
Luis de Bethencourt | 447c1b1 | 2015-12-12 20:07:32 +0000 | [diff] [blame] | 54 | * taken from a "face" event such as the one generated from the 'facedetect' |
| 55 | * element. The Alpha channel values should be one of the following (cv.hpp): |
| 56 | * enum{ |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 57 | * GC_BGD = 0, //!< background |
| 58 | * GC_FGD = 1, //!< foreground |
| 59 | * GC_PR_BGD = 2, //!< most probably background |
| 60 | * GC_PR_FGD = 3 //!< most probably foreground |
| 61 | * }; |
| 62 | * with values over GC_PR_FGD interpreted as GC_PR_FGD. IN CASE OF no alpha mask |
Luis de Bethencourt | 447c1b1 | 2015-12-12 20:07:32 +0000 | [diff] [blame] | 63 | * input (all 0's or all 1's), the 'GstOpenCvFaceDetect-face' downstream event |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 64 | * is used to create a bbox of PR_FG elements. If both foreground alpha |
| 65 | * is not specified and there is no face detection, nothing is done. |
| 66 | * |
Luis de Bethencourt | 447c1b1 | 2015-12-12 20:07:32 +0000 | [diff] [blame] | 67 | * [1] C. Rother, V. Kolmogorov, and A. Blake, "GrabCut: Interactive foreground |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 68 | * extraction using iterated graph cuts, ACM Trans. Graph., vol. 23, pp. 309–314, |
| 69 | * 2004. |
| 70 | * |
| 71 | * <refsect2> |
| 72 | * <title>Example launch line</title> |
| 73 | * |[ |
| 74 | * gst-launch-1.0 --gst-debug=grabcut=4 v4l2src device=/dev/video0 ! videoconvert ! grabcut ! videoconvert ! video/x-raw,width=320,height=240 ! ximagesink |
| 75 | * ]| |
| 76 | * Another example launch line |
| 77 | * |[ |
| 78 | * gst-launch-1.0 --gst-debug=grabcut=4 v4l2src device=/dev/video0 ! videoconvert ! facedetect display=0 ! videoconvert ! grabcut test-mode=true ! videoconvert ! video/x-raw,width=320,height=240 ! ximagesink |
| 79 | * ]| |
| 80 | * </refsect2> |
| 81 | */ |
| 82 | |
| 83 | #ifdef HAVE_CONFIG_H |
| 84 | #include <config.h> |
| 85 | #endif |
| 86 | |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 87 | #include "gstgrabcut.h" |
| 88 | extern "C" |
| 89 | { |
Руслан Ижбулатов | 589407a | 2015-04-11 16:53:38 +0000 | [diff] [blame] | 90 | #include <opencv2/imgproc/imgproc_c.h> |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 91 | } |
Руслан Ижбулатов | 589407a | 2015-04-11 16:53:38 +0000 | [diff] [blame] | 92 | #include <opencv2/imgproc/imgproc.hpp> |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 93 | GST_DEBUG_CATEGORY_STATIC (gst_grabcut_debug); |
| 94 | #define GST_CAT_DEFAULT gst_grabcut_debug |
| 95 | |
Vanessa Chipirras Navalon | 0a08e77 | 2016-01-27 10:05:13 +0100 | [diff] [blame] | 96 | using namespace cv; |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 97 | /* Filter signals and args */ |
| 98 | enum |
| 99 | { |
| 100 | /* FILL ME */ |
| 101 | LAST_SIGNAL |
| 102 | }; |
| 103 | |
| 104 | enum |
| 105 | { |
| 106 | PROP_0, |
| 107 | PROP_TEST_MODE, |
| 108 | PROP_SCALE |
| 109 | }; |
| 110 | |
| 111 | #define DEFAULT_TEST_MODE FALSE |
| 112 | #define DEFAULT_SCALE 1.6 |
| 113 | |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 114 | G_DEFINE_TYPE (GstGrabcut, gst_grabcut, GST_TYPE_OPENCV_VIDEO_FILTER); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 115 | static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", |
| 116 | GST_PAD_SINK, |
| 117 | GST_PAD_ALWAYS, |
| 118 | GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA"))); |
| 119 | |
| 120 | static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", |
| 121 | GST_PAD_SRC, |
| 122 | GST_PAD_ALWAYS, |
| 123 | GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA"))); |
| 124 | |
| 125 | |
| 126 | static void gst_grabcut_set_property (GObject * object, guint prop_id, |
| 127 | const GValue * value, GParamSpec * pspec); |
| 128 | static void gst_grabcut_get_property (GObject * object, guint prop_id, |
| 129 | GValue * value, GParamSpec * pspec); |
| 130 | |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 131 | static GstFlowReturn gst_grabcut_transform_ip (GstOpencvVideoFilter * filter, |
| 132 | GstBuffer * buf, IplImage * img); |
| 133 | static gboolean gst_grabcut_set_caps (GstOpencvVideoFilter * filter, |
| 134 | gint in_width, gint in_height, gint in_depth, gint in_channels, |
| 135 | gint out_width, gint out_height, gint out_depth, gint out_channels); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 136 | |
| 137 | static void gst_grabcut_release_all_pointers (GstGrabcut * filter); |
| 138 | |
| 139 | static gboolean gst_grabcut_stop (GstBaseTransform * basesrc); |
| 140 | static void compose_matrix_from_image (CvMat * output, IplImage * input); |
| 141 | |
| 142 | static int initialise_grabcut (struct grabcut_params *GC, IplImage * image_c, |
| 143 | CvMat * mask_c); |
| 144 | static int run_grabcut_iteration (struct grabcut_params *GC, |
| 145 | IplImage * image_c, CvMat * mask_c, CvRect * bbox); |
| 146 | static int run_grabcut_iteration2 (struct grabcut_params *GC, |
| 147 | IplImage * image_c, CvMat * mask_c, CvRect * bbox); |
| 148 | static int finalise_grabcut (struct grabcut_params *GC); |
| 149 | |
| 150 | /* initialize the grabcut's class */ |
| 151 | static void |
| 152 | gst_grabcut_class_init (GstGrabcutClass * klass) |
| 153 | { |
| 154 | GObjectClass *gobject_class = (GObjectClass *) klass; |
| 155 | GstElementClass *element_class = GST_ELEMENT_CLASS (klass); |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 156 | GstOpencvVideoFilterClass *cvbasefilter_class = |
| 157 | (GstOpencvVideoFilterClass *) klass; |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 158 | GstBaseTransformClass *btrans_class = (GstBaseTransformClass *) klass; |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 159 | |
| 160 | gobject_class->set_property = gst_grabcut_set_property; |
| 161 | gobject_class->get_property = gst_grabcut_get_property; |
| 162 | |
| 163 | btrans_class->stop = gst_grabcut_stop; |
| 164 | btrans_class->passthrough_on_same_caps = TRUE; |
| 165 | |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 166 | cvbasefilter_class->cv_trans_ip_func = gst_grabcut_transform_ip; |
| 167 | cvbasefilter_class->cv_set_caps = gst_grabcut_set_caps; |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 168 | |
| 169 | g_object_class_install_property (gobject_class, PROP_TEST_MODE, |
| 170 | g_param_spec_boolean ("test-mode", "test-mode", |
| 171 | "If true, the output RGB is overwritten with the segmented foreground. Alpha channel same as normal case ", |
| 172 | DEFAULT_TEST_MODE, (GParamFlags) |
| 173 | (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); |
| 174 | |
| 175 | g_object_class_install_property (gobject_class, PROP_SCALE, |
| 176 | g_param_spec_float ("scale", "scale", |
| 177 | "Grow factor for the face bounding box, if present", 1.0, |
| 178 | 4.0, DEFAULT_SCALE, |
| 179 | (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); |
| 180 | |
| 181 | gst_element_class_set_static_metadata (element_class, |
| 182 | "Grabcut-based image FG/BG segmentation", "Filter/Effect/Video", |
| 183 | "Runs Grabcut algorithm on input alpha. Values: BG=0, FG=1, PR_BG=2, PR_FGD=3; \ |
| 184 | NOTE: larger values of alpha (notably 255) are interpreted as PR_FGD too. \n\ |
| 185 | IN CASE OF no alpha mask input (all 0's or all 1's), the 'face' \ |
| 186 | downstream event is used to create a bbox of PR_FG elements.\n\ |
| 187 | IF nothing is present, then nothing is done.", "Miguel Casas-Sanchez <miguelecasassanchez@gmail.com>"); |
| 188 | |
Vineeth TM | 8cdfb13 | 2016-03-04 15:50:26 +0900 | [diff] [blame] | 189 | gst_element_class_add_static_pad_template (element_class, &src_factory); |
| 190 | gst_element_class_add_static_pad_template (element_class, &sink_factory); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 191 | } |
| 192 | |
| 193 | |
| 194 | /* initialize the new element |
| 195 | * instantiate pads and add them to element |
| 196 | * set pad calback functions |
| 197 | * initialize instance structure |
| 198 | */ |
| 199 | static void |
| 200 | gst_grabcut_init (GstGrabcut * filter) |
| 201 | { |
| 202 | filter->test_mode = DEFAULT_TEST_MODE; |
| 203 | filter->scale = DEFAULT_SCALE; |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 204 | gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER (filter), |
| 205 | TRUE); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 206 | } |
| 207 | |
| 208 | |
| 209 | static void |
| 210 | gst_grabcut_set_property (GObject * object, guint prop_id, |
| 211 | const GValue * value, GParamSpec * pspec) |
| 212 | { |
| 213 | GstGrabcut *grabcut = GST_GRABCUT (object); |
| 214 | |
| 215 | switch (prop_id) { |
| 216 | case PROP_TEST_MODE: |
| 217 | grabcut->test_mode = g_value_get_boolean (value); |
| 218 | break; |
| 219 | case PROP_SCALE: |
| 220 | grabcut->scale = g_value_get_float (value); |
| 221 | break; |
| 222 | default: |
| 223 | G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); |
| 224 | break; |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | static void |
| 229 | gst_grabcut_get_property (GObject * object, guint prop_id, |
| 230 | GValue * value, GParamSpec * pspec) |
| 231 | { |
| 232 | GstGrabcut *filter = GST_GRABCUT (object); |
| 233 | |
| 234 | switch (prop_id) { |
| 235 | case PROP_TEST_MODE: |
| 236 | g_value_set_boolean (value, filter->test_mode); |
| 237 | break; |
| 238 | case PROP_SCALE: |
| 239 | g_value_set_float (value, filter->scale); |
| 240 | break; |
| 241 | default: |
| 242 | G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); |
| 243 | break; |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | /* GstElement vmethod implementations */ |
| 248 | /* this function handles the link with other elements */ |
| 249 | static gboolean |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 250 | gst_grabcut_set_caps (GstOpencvVideoFilter * filter, gint in_width, |
| 251 | gint in_height, gint in_depth, gint in_channels, gint out_width, |
| 252 | gint out_height, gint out_depth, gint out_channels) |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 253 | { |
| 254 | GstGrabcut *grabcut = GST_GRABCUT (filter); |
| 255 | CvSize size; |
| 256 | |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 257 | size = cvSize (in_width, in_height); |
| 258 | |
| 259 | /* If cvRGB is already allocated, it means there's a cap modification, |
| 260 | * so release first all the images. */ |
| 261 | if (!grabcut->cvRGBin) |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 262 | gst_grabcut_release_all_pointers (grabcut); |
| 263 | |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 264 | grabcut->cvRGBin = cvCreateImage (size, IPL_DEPTH_8U, 3); |
| 265 | |
| 266 | grabcut->cvA = cvCreateImage (size, IPL_DEPTH_8U, 1); |
| 267 | grabcut->cvB = cvCreateImage (size, IPL_DEPTH_8U, 1); |
| 268 | grabcut->cvC = cvCreateImage (size, IPL_DEPTH_8U, 1); |
| 269 | grabcut->cvD = cvCreateImage (size, IPL_DEPTH_8U, 1); |
| 270 | |
| 271 | grabcut->grabcut_mask = cvCreateMat (size.height, size.width, CV_8UC1); |
| 272 | cvZero (grabcut->grabcut_mask); |
| 273 | initialise_grabcut (&(grabcut->GC), grabcut->cvRGBin, grabcut->grabcut_mask); |
| 274 | |
| 275 | return TRUE; |
| 276 | } |
| 277 | |
| 278 | /* Clean up */ |
| 279 | static gboolean |
| 280 | gst_grabcut_stop (GstBaseTransform * basesrc) |
| 281 | { |
| 282 | GstGrabcut *filter = GST_GRABCUT (basesrc); |
| 283 | |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 284 | if (filter->cvRGBin != NULL) |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 285 | gst_grabcut_release_all_pointers (filter); |
| 286 | |
| 287 | return TRUE; |
| 288 | } |
| 289 | |
| 290 | static void |
| 291 | gst_grabcut_release_all_pointers (GstGrabcut * filter) |
| 292 | { |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 293 | cvReleaseImage (&filter->cvRGBin); |
| 294 | |
| 295 | cvReleaseImage (&filter->cvA); |
| 296 | cvReleaseImage (&filter->cvB); |
| 297 | cvReleaseImage (&filter->cvC); |
| 298 | cvReleaseImage (&filter->cvD); |
| 299 | |
| 300 | finalise_grabcut (&(filter->GC)); |
| 301 | } |
| 302 | |
| 303 | static GstFlowReturn |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 304 | gst_grabcut_transform_ip (GstOpencvVideoFilter * filter, GstBuffer * buffer, |
| 305 | IplImage * img) |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 306 | { |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 307 | GstGrabcut *gc = GST_GRABCUT (filter); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 308 | gint alphapixels; |
| 309 | |
| 310 | GstVideoRegionOfInterestMeta *meta; |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 311 | meta = gst_buffer_get_video_region_of_interest_meta (buffer); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 312 | if (meta) { |
| 313 | gc->facepos.x = (meta->x) - ((gc->scale - 1) * meta->w / 2); |
| 314 | gc->facepos.y = (meta->y) - ((gc->scale - 1) * meta->h / 2); |
| 315 | gc->facepos.width = meta->w * gc->scale * 0.9; |
| 316 | gc->facepos.height = meta->h * gc->scale * 1.1; |
| 317 | } else { |
| 318 | memset (&(gc->facepos), 0, sizeof (gc->facepos)); |
| 319 | } |
| 320 | |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 321 | /* normally input should be RGBA */ |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 322 | cvSplit (img, gc->cvA, gc->cvB, gc->cvC, gc->cvD); |
| 323 | cvCvtColor (img, gc->cvRGBin, CV_BGRA2BGR); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 324 | compose_matrix_from_image (gc->grabcut_mask, gc->cvD); |
| 325 | |
Luis de Bethencourt | 447c1b1 | 2015-12-12 20:07:32 +0000 | [diff] [blame] | 326 | /* Pass cvD to grabcut_mask for the graphcut stuff but that only if |
| 327 | really there is something in the mask! otherwise -->input bbox is |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 328 | what we use */ |
| 329 | alphapixels = cvCountNonZero (gc->cvD); |
| 330 | if ((0 < alphapixels) && (alphapixels < (gc->width * gc->height))) { |
| 331 | GST_INFO ("running on mask"); |
| 332 | run_grabcut_iteration (&(gc->GC), gc->cvRGBin, gc->grabcut_mask, NULL); |
| 333 | } else { |
| 334 | |
| 335 | if ((abs (gc->facepos.width) > 2) && (abs (gc->facepos.height) > 2)) { |
| 336 | GST_INFO ("running on bbox (%d,%d),(%d,%d)", gc->facepos.x, gc->facepos.y, |
| 337 | gc->facepos.width, gc->facepos.height); |
| 338 | run_grabcut_iteration2 (&(gc->GC), gc->cvRGBin, gc->grabcut_mask, |
| 339 | &(gc->facepos)); |
| 340 | } else { |
| 341 | GST_WARNING ("No face info present, skipping frame."); |
| 342 | return GST_FLOW_OK; |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | /* if we want to display, just overwrite the output */ |
| 347 | if (gc->test_mode) { |
| 348 | /* get only FG, PR_FG */ |
| 349 | cvAndS (gc->grabcut_mask, cvRealScalar (1), gc->grabcut_mask, NULL); |
| 350 | /* (saturated) FG, PR_FG --> 255 */ |
| 351 | cvConvertScale (gc->grabcut_mask, gc->grabcut_mask, 255.0, 0.0); |
| 352 | |
| 353 | cvAnd (gc->grabcut_mask, gc->cvA, gc->cvA, NULL); |
| 354 | cvAnd (gc->grabcut_mask, gc->cvB, gc->cvB, NULL); |
| 355 | cvAnd (gc->grabcut_mask, gc->cvC, gc->cvC, NULL); |
| 356 | } |
| 357 | |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 358 | cvMerge (gc->cvA, gc->cvB, gc->cvC, gc->cvD, img); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 359 | |
| 360 | if (gc->test_mode) { |
Nicolas Dufresne | 2b43312 | 2016-12-03 22:40:11 -0500 | [diff] [blame] | 361 | cvRectangle (img, |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 362 | cvPoint (gc->facepos.x, gc->facepos.y), |
| 363 | cvPoint (gc->facepos.x + gc->facepos.width, |
| 364 | gc->facepos.y + gc->facepos.height), CV_RGB (255, 0, 255), 1, 8, 0); |
| 365 | } |
| 366 | |
| 367 | return GST_FLOW_OK; |
| 368 | } |
| 369 | |
| 370 | /* entry point to initialize the plug-in |
| 371 | * initialize the plug-in itself |
| 372 | * register the element factories and other features |
| 373 | */ |
| 374 | gboolean |
| 375 | gst_grabcut_plugin_init (GstPlugin * plugin) |
| 376 | { |
| 377 | /* debug category for fltering log messages |
| 378 | * |
| 379 | */ |
| 380 | GST_DEBUG_CATEGORY_INIT (gst_grabcut_debug, "grabcut", |
| 381 | 0, |
| 382 | "Grabcut image segmentation on either input alpha or input bounding box"); |
| 383 | |
| 384 | return gst_element_register (plugin, "grabcut", GST_RANK_NONE, |
| 385 | GST_TYPE_GRABCUT); |
| 386 | } |
| 387 | |
| 388 | void |
| 389 | compose_matrix_from_image (CvMat * output, IplImage * input) |
| 390 | { |
| 391 | |
| 392 | int x, y; |
| 393 | for (x = 0; x < output->cols; x++) { |
| 394 | for (y = 0; y < output->rows; y++) { |
| 395 | CV_MAT_ELEM (*output, uchar, y, x) = |
Vanessa Chipirras Navalon | 0a08e77 | 2016-01-27 10:05:13 +0100 | [diff] [blame] | 396 | (cvGetReal2D (input, y, x) <= GC_PR_FGD) ? cvGetReal2D (input, y, |
| 397 | x) : GC_PR_FGD; |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 398 | } |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | |
| 403 | int |
| 404 | initialise_grabcut (struct grabcut_params *GC, IplImage * image_c, |
| 405 | CvMat * mask_c) |
| 406 | { |
Vanessa Chipirras Navalon | 0a08e77 | 2016-01-27 10:05:13 +0100 | [diff] [blame] | 407 | GC->image = (void *) new Mat (cvarrToMat (image_c, false)); /* "true" refers to copydata */ |
| 408 | GC->mask = (void *) new Mat (cvarrToMat (mask_c, false)); |
| 409 | GC->bgdModel = (void *) new Mat (); /* "true" refers to copydata */ |
| 410 | GC->fgdModel = (void *) new Mat (); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 411 | |
| 412 | return (0); |
| 413 | } |
| 414 | |
| 415 | int |
| 416 | run_grabcut_iteration (struct grabcut_params *GC, IplImage * image_c, |
| 417 | CvMat * mask_c, CvRect * bbox) |
| 418 | { |
Vanessa Chipirras Navalon | 0a08e77 | 2016-01-27 10:05:13 +0100 | [diff] [blame] | 419 | ((Mat *) GC->image)->data = (uchar *) image_c->imageData; |
| 420 | ((Mat *) GC->mask)->data = mask_c->data.ptr; |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 421 | |
| 422 | if (cvCountNonZero (mask_c)) |
Vanessa Chipirras Navalon | 0a08e77 | 2016-01-27 10:05:13 +0100 | [diff] [blame] | 423 | grabCut (*((Mat *) GC->image), *((Mat *) GC->mask), Rect (), |
| 424 | *((Mat *) GC->bgdModel), *((Mat *) GC->fgdModel), 1, |
| 425 | GC_INIT_WITH_MASK); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 426 | |
| 427 | return (0); |
| 428 | } |
| 429 | |
| 430 | int |
| 431 | run_grabcut_iteration2 (struct grabcut_params *GC, IplImage * image_c, |
| 432 | CvMat * mask_c, CvRect * bbox) |
| 433 | { |
Vanessa Chipirras Navalon | 0a08e77 | 2016-01-27 10:05:13 +0100 | [diff] [blame] | 434 | ((Mat *) GC->image)->data = (uchar *) image_c->imageData; |
| 435 | ((Mat *) GC->mask)->data = mask_c->data.ptr; |
| 436 | grabCut (*((Mat *) GC->image), *((Mat *) GC->mask), *(bbox), |
| 437 | *((Mat *) GC->bgdModel), *((Mat *) GC->fgdModel), 1, |
| 438 | GC_INIT_WITH_RECT); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 439 | |
| 440 | return (0); |
| 441 | } |
| 442 | |
| 443 | int |
| 444 | finalise_grabcut (struct grabcut_params *GC) |
| 445 | { |
Vanessa Chipirras Navalon | 0a08e77 | 2016-01-27 10:05:13 +0100 | [diff] [blame] | 446 | delete ((Mat *) GC->image); |
| 447 | delete ((Mat *) GC->mask); |
| 448 | delete ((Mat *) GC->bgdModel); |
| 449 | delete ((Mat *) GC->fgdModel); |
Miguel Casas-Sanchez | 7c2177b | 2013-07-17 11:28:28 +0200 | [diff] [blame] | 450 | |
| 451 | return (0); |
| 452 | } |