[Spice-devel,v16,11/23] streaming: Let the video encoder manage the compressed buffer

Submitted by Francois Gouget on June 7, 2016, 1:59 p.m.

Details

Message ID fa417cc25348cbb5589bb55e1878e6e4064e6f7f.1465306176.git.fgouget@free.fr
State New
Headers show
Series "Add GStreamer support for video streaming" ( rev: 22 21 20 19 18 ) in Spice

Not browsing as part of any series.

Commit Message

Francois Gouget June 7, 2016, 1:59 p.m.
This way the video encoder is not forced to use malloc()/free().
This also allows more flexibility in how the video encoder manages the
buffer which allows for a zero-copy implementation in both video
encoders.

Signed-off-by: Francois Gouget <fgouget@codeweavers.com>
---
 server/dcc-send.c          | 24 +++++++------
 server/dcc.c               |  5 ---
 server/dcc.h               |  3 --
 server/gstreamer-encoder.c | 60 +++++++++++++++++++++-----------
 server/mjpeg-encoder.c     | 86 +++++++++++++++++++++++++++++-----------------
 server/video-encoder.h     | 28 +++++++++++----
 6 files changed, 129 insertions(+), 77 deletions(-)

Patch hide | download patch | download mbox

diff --git a/server/dcc-send.c b/server/dcc-send.c
index 497f879..006fdc8 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -1651,6 +1651,12 @@  static void red_lossy_marshall_qxl_draw_text(RedChannelClient *rcc,
     }
 }
 
+static void red_release_video_encoder_buffer(uint8_t *data, void *opaque)
+{
+    VideoBuffer *buffer = (VideoBuffer*)opaque;
+    buffer->free(buffer);
+}
+
 static int red_marshall_stream_data(RedChannelClient *rcc,
                                     SpiceMarshaller *base_marshaller, Drawable *drawable)
 {
@@ -1659,7 +1665,6 @@  static int red_marshall_stream_data(RedChannelClient *rcc,
     Stream *stream = drawable->stream;
     SpiceCopy *copy;
     uint32_t frame_mm_time;
-    uint32_t n;
     int is_sized;
     int ret;
 
@@ -1681,7 +1686,6 @@  static int red_marshall_stream_data(RedChannelClient *rcc,
 
     StreamAgent *agent = &dcc->stream_agents[get_stream_id(display, stream)];
     uint64_t time_now = spice_get_monotonic_time_ns();
-    size_t outbuf_size;
 
     if (!dcc->use_video_encoder_rate_control) {
         if (time_now - agent->last_send_time < (1000 * 1000 * 1000) / agent->fps) {
@@ -1693,18 +1697,17 @@  static int red_marshall_stream_data(RedChannelClient *rcc,
         }
     }
 
+    VideoBuffer *outbuf;
     /* workaround for vga streams */
     frame_mm_time =  drawable->red_drawable->mm_time ?
                         drawable->red_drawable->mm_time :
                         reds_get_mm_time();
-    outbuf_size = dcc->send_data.stream_outbuf_size;
     ret = !agent->video_encoder ? VIDEO_ENCODER_FRAME_UNSUPPORTED :
           agent->video_encoder->encode_frame(agent->video_encoder,
                                              frame_mm_time,
                                              &copy->src_bitmap->u.bitmap,
                                              &copy->src_area, stream->top_down,
-                                             &dcc->send_data.stream_outbuf,
-                                             &outbuf_size, &n);
+                                             &outbuf);
     switch (ret) {
     case VIDEO_ENCODER_FRAME_DROP:
         spice_assert(dcc->use_video_encoder_rate_control);
@@ -1720,7 +1723,6 @@  static int red_marshall_stream_data(RedChannelClient *rcc,
         spice_error("bad return value (%d) from VideoEncoder::encode_frame", ret);
         return FALSE;
     }
-    dcc->send_data.stream_outbuf_size = outbuf_size;
 
     if (!is_sized) {
         SpiceMsgDisplayStreamData stream_data;
@@ -1729,7 +1731,7 @@  static int red_marshall_stream_data(RedChannelClient *rcc,
 
         stream_data.base.id = get_stream_id(display, stream);
         stream_data.base.multi_media_time = frame_mm_time;
-        stream_data.data_size = n;
+        stream_data.data_size = outbuf->size;
 
         spice_marshall_msg_display_stream_data(base_marshaller, &stream_data);
     } else {
@@ -1739,7 +1741,7 @@  static int red_marshall_stream_data(RedChannelClient *rcc,
 
         stream_data.base.id = get_stream_id(display, stream);
         stream_data.base.multi_media_time = frame_mm_time;
-        stream_data.data_size = n;
+        stream_data.data_size = outbuf->size;
         stream_data.width = copy->src_area.right - copy->src_area.left;
         stream_data.height = copy->src_area.bottom - copy->src_area.top;
         stream_data.dest = drawable->red_drawable->bbox;
@@ -1748,12 +1750,12 @@  static int red_marshall_stream_data(RedChannelClient *rcc,
         rect_debug(&stream_data.dest);
         spice_marshall_msg_display_stream_data_sized(base_marshaller, &stream_data);
     }
-    spice_marshaller_add_ref(base_marshaller,
-                             dcc->send_data.stream_outbuf, n);
+    spice_marshaller_add_ref_full(base_marshaller, outbuf->data, outbuf->size,
+                                  &red_release_video_encoder_buffer, outbuf);
     agent->last_send_time = time_now;
 #ifdef STREAM_STATS
     agent->stats.num_frames_sent++;
-    agent->stats.size_sent += n;
+    agent->stats.size_sent += outbuf->size;
     agent->stats.end = frame_mm_time;
 #endif
 
diff --git a/server/dcc.c b/server/dcc.c
index 336bae0..4abbed2 100644
--- a/server/dcc.c
+++ b/server/dcc.c
@@ -381,10 +381,6 @@  DisplayChannelClient *dcc_new(DisplayChannel *display,
     // TODO: tune quality according to bandwidth
     dcc->jpeg_quality = 85;
 
-    size_t stream_buf_size;
-    stream_buf_size = 32*1024;
-    dcc->send_data.stream_outbuf = spice_malloc(stream_buf_size);
-    dcc->send_data.stream_outbuf_size = stream_buf_size;
     dcc->send_data.free_list.res =
         spice_malloc(sizeof(SpiceResourceList) +
                      DISPLAY_FREE_LIST_DEFAULT_SIZE * sizeof(SpiceResourceID));
@@ -492,7 +488,6 @@  void dcc_stop(DisplayChannelClient *dcc)
     dcc->pixmap_cache = NULL;
     dcc_release_glz(dcc);
     dcc_palette_cache_reset(dcc);
-    free(dcc->send_data.stream_outbuf);
     free(dcc->send_data.free_list.res);
     dcc_destroy_stream_agents(dcc);
     dcc_encoders_free(dcc);
diff --git a/server/dcc.h b/server/dcc.h
index a11d25a..d9d5ebf 100644
--- a/server/dcc.h
+++ b/server/dcc.h
@@ -88,9 +88,6 @@  struct DisplayChannelClient {
     uint32_t palette_cache_items;
 
     struct {
-        uint32_t stream_outbuf_size;
-        uint8_t *stream_outbuf; // caution stream buffer is also used as compress bufs!!!
-
         FreeList free_list;
         uint64_t pixmap_cache_items[MAX_DRAWABLE_PIXMAP_CACHE_ITEMS];
         int num_pixmap_cache_items;
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index e0b10b3..4838baf 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -37,6 +37,12 @@  typedef struct {
     uint32_t bpp;
 } SpiceFormatForGStreamer;
 
+typedef struct SpiceGstVideoBuffer {
+    VideoBuffer base;
+    GstBuffer *gst_buffer;
+    GstMapInfo map;
+} SpiceGstVideoBuffer;
+
 typedef struct SpiceGstEncoder {
     VideoEncoder base;
 
@@ -81,6 +87,26 @@  typedef struct SpiceGstEncoder {
 } SpiceGstEncoder;
 
 
+/* ---------- The SpiceGstVideoBuffer implementation ---------- */
+
+static void spice_gst_video_buffer_free(VideoBuffer *video_buffer)
+{
+    SpiceGstVideoBuffer *buffer = (SpiceGstVideoBuffer*)video_buffer;
+    if (buffer->gst_buffer) {
+        gst_buffer_unmap(buffer->gst_buffer, &buffer->map);
+        gst_buffer_unref(buffer->gst_buffer);
+    }
+    free(buffer);
+}
+
+static SpiceGstVideoBuffer* create_gst_video_buffer(void)
+{
+    SpiceGstVideoBuffer *buffer = spice_new0(SpiceGstVideoBuffer, 1);
+    buffer->base.free = spice_gst_video_buffer_free;
+    return buffer;
+}
+
+
 /* ---------- Miscellaneous SpiceGstEncoder helpers ---------- */
 
 static inline double get_mbps(uint64_t bit_rate)
@@ -471,29 +497,22 @@  static int push_raw_frame(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
 
 /* A helper for spice_gst_encoder_encode_frame() */
 static int pull_compressed_buffer(SpiceGstEncoder *encoder,
-                                  uint8_t **outbuf, size_t *outbuf_size,
-                                  uint32_t *data_size)
+                                  VideoBuffer **outbuf)
 {
-    spice_return_val_if_fail(outbuf && outbuf_size, VIDEO_ENCODER_FRAME_UNSUPPORTED);
-
     GstSample *sample = gst_app_sink_pull_sample(encoder->appsink);
     if (sample) {
-        GstMapInfo map;
-        GstBuffer *buffer = gst_sample_get_buffer(sample);
-        if (buffer && gst_buffer_map(buffer, &map, GST_MAP_READ)) {
-            gint size = gst_buffer_get_size(buffer);
-            if (!*outbuf || *outbuf_size < size) {
-                free(*outbuf);
-                *outbuf = spice_malloc(size);
-                *outbuf_size = size;
-            }
-            /* TODO Try to avoid this copy by changing the GstBuffer handling */
-            memcpy(*outbuf, map.data, size);
-            *data_size = size;
-            gst_buffer_unmap(buffer, &map);
+        SpiceGstVideoBuffer *buffer = create_gst_video_buffer();
+        buffer->gst_buffer = gst_sample_get_buffer(sample);
+        if (buffer->gst_buffer &&
+            gst_buffer_map(buffer->gst_buffer, &buffer->map, GST_MAP_READ)) {
+            buffer->base.data = buffer->map.data;
+            buffer->base.size = gst_buffer_get_size(buffer->gst_buffer);
+            *outbuf = (VideoBuffer*)buffer;
+            gst_buffer_ref(buffer->gst_buffer);
             gst_sample_unref(sample);
             return VIDEO_ENCODER_FRAME_ENCODE_DONE;
         }
+        buffer->base.free((VideoBuffer*)buffer);
         gst_sample_unref(sample);
     }
     spice_debug("failed to pull the compressed buffer");
@@ -514,10 +533,11 @@  static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
                                           uint32_t frame_mm_time,
                                           const SpiceBitmap *bitmap,
                                           const SpiceRect *src, int top_down,
-                                          uint8_t **outbuf, size_t *outbuf_size,
-                                          uint32_t *data_size)
+                                          VideoBuffer **outbuf)
 {
     SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+    g_return_val_if_fail(outbuf != NULL, VIDEO_ENCODER_FRAME_UNSUPPORTED);
+    *outbuf = NULL;
 
     uint32_t width = src->right - src->left;
     uint32_t height = src->bottom - src->top;
@@ -545,7 +565,7 @@  static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
 
     int rc = push_raw_frame(encoder, bitmap, src, top_down);
     if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
-        rc = pull_compressed_buffer(encoder, outbuf, outbuf_size, data_size);
+        rc = pull_compressed_buffer(encoder, outbuf);
         if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
             /* The input buffer will be stuck in the pipeline, preventing
              * later ones from being processed. Furthermore something went
diff --git a/server/mjpeg-encoder.c b/server/mjpeg-encoder.c
index 5c143a6..2db6ca9 100644
--- a/server/mjpeg-encoder.c
+++ b/server/mjpeg-encoder.c
@@ -70,6 +70,9 @@  static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
  */
 #define MJPEG_WARMUP_TIME (NSEC_PER_SEC * 3)
 
+/* The compressed buffer initial size. */
+#define MJPEG_INITIAL_BUFFER_SIZE (32 * 1024)
+
 enum {
     MJPEG_QUALITY_EVAL_TYPE_SET,
     MJPEG_QUALITY_EVAL_TYPE_UPGRADE,
@@ -154,6 +157,11 @@  typedef struct MJpegEncoderRateControl {
     uint64_t warmup_start_time;
 } MJpegEncoderRateControl;
 
+typedef struct MJpegVideoBuffer {
+    VideoBuffer base;
+    size_t maxsize;
+} MJpegVideoBuffer;
+
 typedef struct MJpegEncoder {
     VideoEncoder base;
     uint8_t *row;
@@ -180,6 +188,26 @@  static uint32_t get_min_required_playback_delay(uint64_t frame_enc_size,
                                                 uint64_t byte_rate,
                                                 uint32_t latency);
 
+static void mjpeg_video_buffer_free(VideoBuffer *video_buffer)
+{
+    MJpegVideoBuffer *buffer = (MJpegVideoBuffer*)video_buffer;
+    free(buffer->base.data);
+    free(buffer);
+}
+
+static MJpegVideoBuffer* create_mjpeg_video_buffer(void)
+{
+    MJpegVideoBuffer *buffer = spice_new0(MJpegVideoBuffer, 1);
+    buffer->base.free = mjpeg_video_buffer_free;
+    buffer->maxsize = MJPEG_INITIAL_BUFFER_SIZE;
+    buffer->base.data = malloc(buffer->maxsize);
+    if (!buffer->base.data) {
+        free(buffer);
+        buffer = NULL;
+    }
+    return buffer;
+}
+
 static inline int rate_control_is_active(MJpegEncoder* encoder)
 {
     return encoder->cbs.get_roundtrip_ms != NULL;
@@ -283,24 +311,22 @@  static void term_mem_destination(j_compress_ptr cinfo)
 
 /*
  * Prepare for output to a memory buffer.
- * The caller may supply an own initial buffer with appropriate size.
- * Otherwise, or when the actual data output exceeds the given size,
- * the library adapts the buffer size as necessary.
- * The standard library functions malloc/free are used for allocating
- * larger memory, so the buffer is available to the application after
- * finishing compression, and then the application is responsible for
- * freeing the requested memory.
+ * The caller must supply its own initial buffer and size.
+ * When the actual data output exceeds the given size, the library
+ * will adapt the buffer size as necessary using the malloc()/free()
+ * functions. The buffer is available to the application after the
+ * compression and the application is then responsible for freeing it.
  */
-
 static void
 spice_jpeg_mem_dest(j_compress_ptr cinfo,
                     unsigned char ** outbuffer, size_t * outsize)
 {
   mem_destination_mgr *dest;
-#define OUTPUT_BUF_SIZE  4096 /* choose an efficiently fwrite'able size */
 
-  if (outbuffer == NULL || outsize == NULL) /* sanity check */
+  if (outbuffer == NULL || *outbuffer == NULL ||
+      outsize == NULL || *outsize == 0) { /* sanity check */
     ERREXIT(cinfo, JERR_BUFFER_SIZE);
+  }
 
   /* The destination object is made permanent so that multiple JPEG images
    * can be written to the same buffer without re-executing jpeg_mem_dest.
@@ -315,13 +341,6 @@  spice_jpeg_mem_dest(j_compress_ptr cinfo,
   dest->pub.term_destination = term_mem_destination;
   dest->outbuffer = outbuffer;
   dest->outsize = outsize;
-  if (*outbuffer == NULL || *outsize == 0) {
-    /* Allocate initial buffer */
-    *outbuffer = malloc(OUTPUT_BUF_SIZE);
-    if (*outbuffer == NULL)
-      ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 10);
-    *outsize = OUTPUT_BUF_SIZE;
-  }
 
   dest->pub.next_output_byte = dest->buffer = *outbuffer;
   dest->pub.free_in_buffer = dest->bufsize = *outsize;
@@ -707,7 +726,7 @@  static void mjpeg_encoder_adjust_fps(MJpegEncoder *encoder, uint64_t now)
 static int mjpeg_encoder_start_frame(MJpegEncoder *encoder,
                                      SpiceBitmapFmt format,
                                      const SpiceRect *src,
-                                     uint8_t **dest, size_t *dest_len,
+                                     MJpegVideoBuffer *buffer,
                                      uint32_t frame_mm_time)
 {
     uint32_t quality;
@@ -791,7 +810,8 @@  static int mjpeg_encoder_start_frame(MJpegEncoder *encoder,
         }
     }
 
-    spice_jpeg_mem_dest(&encoder->cinfo, dest, dest_len);
+    spice_jpeg_mem_dest(&encoder->cinfo, &buffer->base.data, &buffer->maxsize);
+
     jpeg_set_defaults(&encoder->cinfo);
     encoder->cinfo.dct_method       = JDCT_IFAST;
     quality = mjpeg_quality_samples[encoder->rate_control.quality_id];
@@ -928,25 +948,29 @@  static int mjpeg_encoder_encode_frame(VideoEncoder *video_encoder,
                                       uint32_t frame_mm_time,
                                       const SpiceBitmap *bitmap,
                                       const SpiceRect *src, int top_down,
-                                      uint8_t **outbuf, size_t *outbuf_size,
-                                      uint32_t *data_size)
+                                      VideoBuffer **outbuf)
 {
     MJpegEncoder *encoder = (MJpegEncoder*)video_encoder;
+    MJpegVideoBuffer *buffer = create_mjpeg_video_buffer();
+    if (!buffer) {
+        return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+    }
 
     int ret = mjpeg_encoder_start_frame(encoder, bitmap->format, src,
-                                        outbuf, outbuf_size,
-                                        frame_mm_time);
-    if (ret != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
-        return ret;
+                                        buffer, frame_mm_time);
+    if (ret == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+        if (encode_frame(encoder, src, bitmap, top_down)) {
+            buffer->base.size = mjpeg_encoder_end_frame(encoder);
+            *outbuf = (VideoBuffer*)buffer;
+        } else {
+            ret = VIDEO_ENCODER_FRAME_UNSUPPORTED;
+        }
     }
 
-    if (!encode_frame(encoder, src, bitmap, top_down)) {
-        return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+    if (ret != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+        buffer->base.free((VideoBuffer*)buffer);
     }
-
-    *data_size = mjpeg_encoder_end_frame(encoder);
-
-    return VIDEO_ENCODER_FRAME_ENCODE_DONE;
+    return ret;
 }
 
 
diff --git a/server/video-encoder.h b/server/video-encoder.h
index 7a04bc6..f4be396 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -24,6 +24,23 @@ 
 #include <inttypes.h>
 #include <common/draw.h>
 
+
+/* A structure containing the data for a compressed frame. See encode_frame(). */
+typedef struct VideoBuffer VideoBuffer;
+struct VideoBuffer {
+    /* A pointer to the compressed frame data. */
+    uint8_t *data;
+
+    /* The size of the compressed frame in bytes. */
+    uint32_t size;
+
+    /* Releases the video buffer resources and deallocates it.
+     *
+     * @buffer:   The video buffer.
+     */
+    void (*free)(VideoBuffer *buffer);
+};
+
 enum {
     VIDEO_ENCODER_FRAME_UNSUPPORTED = -1,
     VIDEO_ENCODER_FRAME_DROP,
@@ -48,11 +65,9 @@  struct VideoEncoder {
      * @bitmap:        A bitmap containing the source video frame.
      * @src:           A rectangle specifying the area occupied by the video.
      * @top_down:      If true the first video line is specified by src.top.
-     * @outbuf:        The buffer for the compressed frame. This must either
-     *                 be NULL or point to a buffer allocated by malloc
-     *                 since it may be reallocated, if its size is too small.
-     * @outbuf_size:   The size of the outbuf buffer.
-     * @data_size:     The size of the compressed frame.
+     * @outbuf:        A pointer to a VideoBuffer structure containing the
+     *                 compressed frame if successful. Call the buffer's
+     *                 free() method as soon as it is no longer needed.
      * @return:
      *     VIDEO_ENCODER_FRAME_ENCODE_DONE if successful.
      *     VIDEO_ENCODER_FRAME_UNSUPPORTED if the frame cannot be encoded.
@@ -62,8 +77,7 @@  struct VideoEncoder {
     int (*encode_frame)(VideoEncoder *encoder, uint32_t frame_mm_time,
                         const SpiceBitmap *bitmap,
                         const SpiceRect *src, int top_down,
-                        uint8_t **outbuf, size_t *outbuf_size,
-                        uint32_t *data_size);
+                        VideoBuffer** outbuf);
 
     /*
      * Bit rate control methods.