diff --git a/edgetpuvision/gst_native.py b/edgetpuvision/gst_native.py
index eb79089..e358d77 100644
--- a/edgetpuvision/gst_native.py
+++ b/edgetpuvision/gst_native.py
@@ -239,16 +239,8 @@
             pts = self.pts
             self.svg = None
 
-        # Note: Buffer IS writable (ref is 1 in native land). However gst-python
-        # took an additional ref so it's now 2 and gst_buffer_is_writable
-        # returns false. We can't modify the buffer without fiddling with refcount.
-        assert buf.mini_object.refcount == 2
-        buf.mini_object.refcount = 1
-        try:
-            self.render_svg(svg, buf)
-            buf.pts = pts
-        finally:
-            buf.mini_object.refcount = 2
+        self.render_svg(svg, buf)
+        buf.pts = pts
 
         with self.cond:
             return self.get_flow_return_locked(Gst.FlowReturn.OK)