aggregator: Reset upstream latency on first buffer

In the case an aggregator is created and pads are requested but only
linked later, we end up never updating the upstream latency.
This was because latency queries on pads that are not linked succeed,
so we never did a new query once a live source has been linked, so the
thread was never started.

https://bugzilla.gnome.org/show_bug.cgi?id=757548
diff --git a/libs/gst/base/gstaggregator.c b/libs/gst/base/gstaggregator.c
index f38d92a..46b3c5d 100644
--- a/libs/gst/base/gstaggregator.c
+++ b/libs/gst/base/gstaggregator.c
@@ -1390,7 +1390,6 @@
   }
 
   GST_DEBUG_OBJECT (element, "Adding pad %s", GST_PAD_NAME (agg_pad));
-  self->priv->has_peer_latency = FALSE;
 
   if (priv->running)
     gst_pad_set_active (GST_PAD (agg_pad), TRUE);
@@ -2165,12 +2164,16 @@
 
   buf_pts = GST_BUFFER_PTS (buffer);
 
-  aggpad->priv->first_buffer = FALSE;
-
   for (;;) {
     SRC_LOCK (self);
     GST_OBJECT_LOCK (self);
     PAD_LOCK (aggpad);
+
+    if (aggpad->priv->first_buffer) {
+      self->priv->has_peer_latency = FALSE;
+      aggpad->priv->first_buffer = FALSE;
+    }
+
     if (gst_aggregator_pad_has_space (self, aggpad)
         && aggpad->priv->flow_return == GST_FLOW_OK) {
       if (head)