diff --git a/gst/rtp/gstrtpvrawpay.c b/gst/rtp/gstrtpvrawpay.c
index 50a4820..d7d66b5 100644
--- a/gst/rtp/gstrtpvrawpay.c
+++ b/gst/rtp/gstrtpvrawpay.c
@@ -325,7 +325,8 @@
   buffers_per_list = packlines_per_list * packets_per_packline;
   buffers_per_list = GST_ROUND_UP_8 (buffers_per_list);
 
-  use_buffer_lists = (rtpvrawpay->chunks_per_frame < (height / yinc));
+  use_buffer_lists = buffers_per_list > 1 &&
+      (rtpvrawpay->chunks_per_frame < (height / yinc));
 
   fields = 1 + interlaced;