diff --git a/plugins/elements/gstinputselector.c b/plugins/elements/gstinputselector.c
index c7c8331..b23b47c 100644
--- a/plugins/elements/gstinputselector.c
+++ b/plugins/elements/gstinputselector.c
@@ -1578,9 +1578,9 @@
 
 typedef struct
 {
-  guint count;
   gboolean live;
-  GstClockTime min, max;
+  GstClockTime live_min, live_max;
+  GstClockTime non_live_min, non_live_max;
 } LatencyFoldData;
 
 static gboolean
@@ -1604,25 +1604,29 @@
   if (res) {
     gboolean live;
     GstClockTime min, max;
+    GstClockTime *min_store, *max_store;
 
     gst_query_parse_latency (query, &live, &min, &max);
 
     GST_LOG_OBJECT (pad, "got latency live:%s min:%" G_GINT64_FORMAT
         " max:%" G_GINT64_FORMAT, live ? "true" : "false", min, max);
 
-    /* FIXME : Why do we only take values into account if it's live ? */
-    if (live || fold_data->count == 0) {
-      if (min > fold_data->min)
-        fold_data->min = min;
-
-      if (fold_data->max == GST_CLOCK_TIME_NONE)
-        fold_data->max = max;
-      else if (max < fold_data->max)
-        fold_data->max = max;
-
+    if (live) {
+      min_store = &fold_data->live_min;
+      max_store = &fold_data->live_max;
       fold_data->live = live;
+    } else {
+      min_store = &fold_data->non_live_min;
+      max_store = &fold_data->non_live_max;
     }
-    fold_data->count += 1;
+
+    if (min > *min_store)
+      *min_store = min;
+
+    if (*max_store == GST_CLOCK_TIME_NONE)
+      *max_store = max;
+    else if (max < *max_store)
+      *max_store = max;
   } else if (peer) {
     GST_DEBUG_OBJECT (pad, "latency query failed");
     g_value_set_boolean (ret, FALSE);
@@ -1656,10 +1660,9 @@
   g_value_init (&ret, G_TYPE_BOOLEAN);
 
 retry:
-  fold_data.count = 0;
   fold_data.live = FALSE;
-  fold_data.min = 0;
-  fold_data.max = GST_CLOCK_TIME_NONE;
+  fold_data.live_min = fold_data.non_live_min = 0;
+  fold_data.live_max = fold_data.non_live_max = GST_CLOCK_TIME_NONE;
 
   g_value_set_boolean (&ret, TRUE);
   res = gst_iterator_fold (it, query_latency_default_fold, &ret, &fold_data);
@@ -1683,15 +1686,24 @@
 
   query_ret = g_value_get_boolean (&ret);
   if (query_ret) {
-    GST_LOG_OBJECT (pad, "got latency live:%s min:%" G_GINT64_FORMAT
-        " max:%" G_GINT64_FORMAT, fold_data.live ? "true" : "false",
-        fold_data.min, fold_data.max);
+    GstClockTime min, max;
 
-    if (fold_data.min > fold_data.max) {
+    if (fold_data.live) {
+      min = fold_data.live_min;
+      max = fold_data.live_max;
+    } else {
+      min = fold_data.non_live_min;
+      max = fold_data.non_live_max;
+    }
+
+    GST_LOG_OBJECT (pad, "got latency live:%s min:%" G_GINT64_FORMAT
+        " max:%" G_GINT64_FORMAT, fold_data.live ? "true" : "false", min, max);
+
+    if (min > max) {
       GST_ERROR_OBJECT (pad, "minimum latency bigger than maximum latency");
     }
 
-    gst_query_set_latency (query, fold_data.live, fold_data.min, fold_data.max);
+    gst_query_set_latency (query, fold_data.live, min, max);
   } else {
     GST_LOG_OBJECT (pad, "latency query failed");
   }