diff --git a/plugins/elements/gstinputselector.c b/plugins/elements/gstinputselector.c
index b23b47c..0359817 100644
--- a/plugins/elements/gstinputselector.c
+++ b/plugins/elements/gstinputselector.c
@@ -1579,8 +1579,7 @@
 typedef struct
 {
   gboolean live;
-  GstClockTime live_min, live_max;
-  GstClockTime non_live_min, non_live_max;
+  GstClockTime min, max;
 } LatencyFoldData;
 
 static gboolean
@@ -1604,7 +1603,6 @@
   if (res) {
     gboolean live;
     GstClockTime min, max;
-    GstClockTime *min_store, *max_store;
 
     gst_query_parse_latency (query, &live, &min, &max);
 
@@ -1612,21 +1610,15 @@
         " max:%" G_GINT64_FORMAT, live ? "true" : "false", min, max);
 
     if (live) {
-      min_store = &fold_data->live_min;
-      max_store = &fold_data->live_max;
+      if (min > fold_data->min)
+        fold_data->min = min;
+
+      if (fold_data->max == GST_CLOCK_TIME_NONE)
+        fold_data->max = max;
+      else if (max < fold_data->max)
+        fold_data->max = max;
       fold_data->live = live;
-    } else {
-      min_store = &fold_data->non_live_min;
-      max_store = &fold_data->non_live_max;
     }
-
-    if (min > *min_store)
-      *min_store = min;
-
-    if (*max_store == GST_CLOCK_TIME_NONE)
-      *max_store = max;
-    else if (max < *max_store)
-      *max_store = max;
   } else if (peer) {
     GST_DEBUG_OBJECT (pad, "latency query failed");
     g_value_set_boolean (ret, FALSE);
@@ -1661,8 +1653,8 @@
 
 retry:
   fold_data.live = FALSE;
-  fold_data.live_min = fold_data.non_live_min = 0;
-  fold_data.live_max = fold_data.non_live_max = GST_CLOCK_TIME_NONE;
+  fold_data.min = 0;
+  fold_data.max = GST_CLOCK_TIME_NONE;
 
   g_value_set_boolean (&ret, TRUE);
   res = gst_iterator_fold (it, query_latency_default_fold, &ret, &fold_data);
@@ -1686,24 +1678,15 @@
 
   query_ret = g_value_get_boolean (&ret);
   if (query_ret) {
-    GstClockTime min, max;
-
-    if (fold_data.live) {
-      min = fold_data.live_min;
-      max = fold_data.live_max;
-    } else {
-      min = fold_data.non_live_min;
-      max = fold_data.non_live_max;
-    }
-
     GST_LOG_OBJECT (pad, "got latency live:%s min:%" G_GINT64_FORMAT
-        " max:%" G_GINT64_FORMAT, fold_data.live ? "true" : "false", min, max);
+        " max:%" G_GINT64_FORMAT, fold_data.live ? "true" : "false",
+        fold_data.min, fold_data.max);
 
-    if (min > max) {
+    if (fold_data.min > fold_data.max) {
       GST_ERROR_OBJECT (pad, "minimum latency bigger than maximum latency");
     }
 
-    gst_query_set_latency (query, fold_data.live, min, max);
+    gst_query_set_latency (query, fold_data.live, fold_data.min, fold_data.max);
   } else {
     GST_LOG_OBJECT (pad, "latency query failed");
   }