Implement proper timestamping for references

Reference frames are now identified using their timestamp:
set the timestamp when queuing the output buffer and use it to identify
the frame later on.

Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com>

src/h264.c

@@ -186,12 +186,15 @@ static void h264_fill_dpb(struct request_data *data,
 		struct h264_dpb_entry *entry = &context->dpb.entries[i];
 		struct object_surface *surface =
 			SURFACE(data, entry->pic.picture_id);
+		uint64_t timestamp;
 
 		if (!entry->valid)
 			continue;
 
-		if (surface)
-			dpb->buf_index = surface->destination_index;
+		if (surface) {
+			timestamp = v4l2_timeval_to_ns(&surface->timestamp);
+			dpb->timestamp = timestamp;
+		}
 
 		dpb->frame_num = entry->pic.frame_idx;
 		dpb->top_field_order_cnt = entry->pic.TopFieldOrderCnt;

src/h265.c

@@ -272,6 +272,8 @@ static void h265_fill_slice_params(VAPictureParameterBufferHEVC *picture,
 	num_rps_poc_lt_curr = 0;
 
 	for (i = 0; i < 15 && slice_type != V4L2_HEVC_SLICE_TYPE_I ; i++) {
+		uint64_t timestamp;
+
 		hevc_picture = &picture->ReferenceFrames[i];
 
 		if (hevc_picture->picture_id == VA_INVALID_SURFACE ||
@@ -284,8 +286,8 @@ static void h265_fill_slice_params(VAPictureParameterBufferHEVC *picture,
 		if (surface_object == NULL)
 			break;
 
-		slice_params->dpb[i].buffer_index =
-			surface_object->destination_index;
+		timestamp = v4l2_timeval_to_ns(&surface_object->timestamp);
+		slice_params->dpb[i].timestamp = timestamp;
 
 		if ((hevc_picture->flags & VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE) != 0) {
 			slice_params->dpb[i].rps =

src/mpeg2.c

@@ -54,6 +54,7 @@ int mpeg2_set_controls(struct request_data *driver_data,
 	struct v4l2_ctrl_mpeg2_quantization quantization;
 	struct object_surface *forward_reference_surface;
 	struct object_surface *backward_reference_surface;
+	uint64_t timestamp;
 	unsigned int i;
 	int rc;
 
@@ -102,21 +103,19 @@ int mpeg2_set_controls(struct request_data *driver_data,
 
 	forward_reference_surface =
 		SURFACE(driver_data, picture->forward_reference_picture);
-	if (forward_reference_surface != NULL)
-		slice_params.forward_ref_index =
-			forward_reference_surface->destination_index;
-	else
-		slice_params.forward_ref_index =
-			surface_object->destination_index;
+	if (forward_reference_surface == NULL)
+		forward_reference_surface = surface_object;
+
+	timestamp = v4l2_timeval_to_ns(&forward_reference_surface->timestamp);
+	slice_params.forward_ref_ts = timestamp;
 
 	backward_reference_surface =
 		SURFACE(driver_data, picture->backward_reference_picture);
-	if (backward_reference_surface != NULL)
-		slice_params.backward_ref_index =
-			backward_reference_surface->destination_index;
-	else
-		slice_params.backward_ref_index =
-			surface_object->destination_index;
+	if (backward_reference_surface == NULL)
+		backward_reference_surface = surface_object;
+
+	timestamp = v4l2_timeval_to_ns(&backward_reference_surface->timestamp);
+	slice_params.backward_ref_ts = timestamp;
 
 	rc = v4l2_set_control(driver_data->video_fd, surface_object->request_fd,
 			      V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS,

src/picture.c

@@ -296,6 +296,8 @@ VAStatus RequestEndPicture(VADriverContextP context, VAContextID context_id)
 	if (surface_object == NULL)
 		return VA_STATUS_ERROR_INVALID_SURFACE;
 
+	gettimeofday(&surface_object->timestamp, NULL);
+
 	request_fd = surface_object->request_fd;
 	if (request_fd < 0) {
 		request_fd = media_request_alloc(driver_data->media_fd);
@@ -310,13 +312,14 @@ VAStatus RequestEndPicture(VADriverContextP context, VAContextID context_id)
 	if (rc != VA_STATUS_SUCCESS)
 		return rc;
 
-	rc = v4l2_queue_buffer(driver_data->video_fd, -1, capture_type,
+	rc = v4l2_queue_buffer(driver_data->video_fd, -1, capture_type, NULL,
 			       surface_object->destination_index, 0,
 			       surface_object->destination_buffers_count);
 	if (rc < 0)
 		return VA_STATUS_ERROR_OPERATION_FAILED;
 
 	rc = v4l2_queue_buffer(driver_data->video_fd, request_fd, output_type,
+			       &surface_object->timestamp,
 			       surface_object->source_index,
 			       surface_object->slices_size, 1);
 	if (rc < 0)

src/surface.h

@@ -62,6 +62,8 @@ struct object_surface {
 	unsigned int slices_size;
 	unsigned int slices_count;
 
+	struct timeval timestamp;
+
 	union {
 		struct {
 			VAPictureParameterBufferMPEG2 picture;

src/v4l2.c

@@ -330,8 +330,8 @@ int v4l2_request_buffers(int video_fd, unsigned int type,
 }
 
 int v4l2_queue_buffer(int video_fd, int request_fd, unsigned int type,
-		      unsigned int index, unsigned int size,
-		      unsigned int buffers_count)
+		      struct timeval *timestamp, unsigned int index,
+		      unsigned int size, unsigned int buffers_count)
 {
 	struct v4l2_plane planes[buffers_count];
 	struct v4l2_buffer buffer;
@@ -358,6 +358,9 @@ int v4l2_queue_buffer(int video_fd, int request_fd, unsigned int type,
 		buffer.request_fd = request_fd;
 	}
 
+	if (timestamp != NULL)
+		buffer.timestamp = *timestamp;
+
 	rc = ioctl(video_fd, VIDIOC_QBUF, &buffer);
 	if (rc < 0) {
 		request_log("Unable to queue buffer: %s\n", strerror(errno));

src/v4l2.h

@@ -47,8 +47,8 @@ int v4l2_query_buffer(int video_fd, unsigned int type, unsigned int index,
 int v4l2_request_buffers(int video_fd, unsigned int type,
 			 unsigned int buffers_count);
 int v4l2_queue_buffer(int video_fd, int request_fd, unsigned int type,
-		      unsigned int index, unsigned int size,
-		      unsigned int buffers_count);
+		      struct timeval *timestamp, unsigned int index,
+		      unsigned int size, unsigned int buffers_count);
 int v4l2_dequeue_buffer(int video_fd, int request_fd, unsigned int type,
 			unsigned int index, unsigned int buffers_count);
 int v4l2_export_buffer(int video_fd, unsigned int type, unsigned int index,