[Spice-devel,v16,21/23] streaming: Add support for GStreamer 0.10

Submitted by Francois Gouget on June 7, 2016, 2 p.m.

Details

Message ID ab62a3cb5685e331d4108166c541019018579276.1465306176.git.fgouget@free.fr
State New
Headers show
Series "Add GStreamer support for video streaming" ( rev: 22 21 20 19 18 ) in Spice

Not browsing as part of any series.

Commit Message

Francois Gouget June 7, 2016, 2 p.m.
configure will use GStreamer 1.0 if present and fall back to
GStreamer 0.10 otherwise.
ffenc_mjpeg takes its bitrate as a long so extend set_gstenc_bitrate().

Signed-off-by: Francois Gouget <fgouget@codeweavers.com>
---
 configure.ac               |  36 ++++++++++---
 server/Makefile.am         |   8 +++
 server/gstreamer-encoder.c | 132 ++++++++++++++++++++++++++++++++++++++-------
 server/reds.c              |   2 +-
 server/video-encoder.h     |   2 +-
 5 files changed, 151 insertions(+), 29 deletions(-)

Patch hide | download patch | download mbox

diff --git a/configure.ac b/configure.ac
index 1cc9f4f..8e3ff47 100644
--- a/configure.ac
+++ b/configure.ac
@@ -69,28 +69,48 @@  dnl Check optional features
 SPICE_CHECK_SMARTCARD
 
 AC_ARG_ENABLE(gstreamer,
-              AS_HELP_STRING([--enable-gstreamer=@<:@auto/yes/no@:>@],
-                             [Enable GStreamer 1.0 support]),,
+              AS_HELP_STRING([--enable-gstreamer=@<:@auto/0.10/1.0/yes/no@:>@],
+                             [Enable GStreamer support]),,
               [enable_gstreamer="auto"])
 
-if test "x$enable_gstreamer" != "xno"; then
+if test "x$enable_gstreamer" != "xno" && test "x$enable_gstreamer" != "x0.10"; then
     SPICE_CHECK_GSTREAMER(GSTREAMER_1_0, 1.0, [gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-video-1.0],
-        [enable_gstreamer="yes"
+        [enable_gstreamer="1.0"
          SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-base 1.0], [appsrc videoconvert appsink])
          SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gstreamer-libav 1.0], [avenc_mjpeg])
          SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-good 1.0], [vp8enc])
          SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-ugly 1.0], [x264enc])
          ],
-         [if test "x$enable_gstreamer" = "xyes"; then
+         [if test "x$enable_gstreamer" = "x1.0"; then
               AC_MSG_ERROR([GStreamer 1.0 support requested but not found. You may set GSTREAMER_1_0_CFLAGS and GSTREAMER_1_0_LIBS to avoid the need to call pkg-config.])
           fi
     ])
 fi
 AM_CONDITIONAL(HAVE_GSTREAMER_1_0, test "x$have_gstreamer_1_0" = "xyes")
 
-if test x"$gstreamer_missing" != x; then
-    SPICE_WARNING([The following GStreamer $enable_gstreamer tools/elements are missing:$gstreamer_missing. The GStreamer video encoder can be built but may not work.])
+if test "x$enable_gstreamer" != "xno" && test "x$enable_gstreamer" != "x1.0"; then
+    SPICE_CHECK_GSTREAMER(GSTREAMER_0_10, 0.10, [gstreamer-0.10 gstreamer-base-0.10 gstreamer-app-0.10 gstreamer-video-0.10],
+        [enable_gstreamer="0.10"
+         SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gst-plugins-base 0.10], [appsrc appsink])
+         SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gstreamer-ffmpeg 0.10], [ffmpegcolorspace ffenc_mjpeg])
+         SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gst-plugins-bad 0.10], [vp8enc])
+         SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gst-plugins-ugly 0.10], [x264enc])
+        ],
+        [if test "x$enable_gstreamer" = "x0.10"; then
+             AC_MSG_ERROR([GStreamer 0.10 support requested but not found. You may set GSTREAMER_0_10_CFLAGS and GSTREAMER_0_10_LIBS to avoid the need to call pkg-config.])
+         fi
+    ])
 fi
+AM_CONDITIONAL(HAVE_GSTREAMER_0_10, test "x$have_gstreamer_0_10" = "xyes")
+
+AS_IF([test "x$enable_gstreamer" = "xyes"],
+      [AC_MSG_ERROR("GStreamer support requested but not found")],
+      [test "x$enable_gstreamer" = "xauto"],
+      [enable_gstreamer="no"
+])
+AS_IF([test x"$missing_gstreamer_elements" = xyes],
+    [SPICE_WARNING([The GStreamer video encoder can be built but may not work.])
+])
 
 AC_ARG_ENABLE([automated_tests],
               AS_HELP_STRING([--enable-automated-tests], [Enable automated tests using spicy-screenshot (part of spice-gtk)]),,
@@ -268,7 +288,7 @@  AC_MSG_NOTICE([
 
         LZ4 support:              ${enable_lz4}
         Smartcard:                ${have_smartcard}
-        GStreamer 1.0:            ${have_gstreamer_1_0}
+        GStreamer:                ${enable_gstreamer}
         SASL support:             ${have_sasl}
         Automated tests:          ${enable_automated_tests}
         Manual:                   ${have_asciidoc}
diff --git a/server/Makefile.am b/server/Makefile.am
index ea3e2ff..09710eb 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -12,6 +12,7 @@  AM_CPPFLAGS =					\
 	$(SASL_CFLAGS)				\
 	$(SLIRP_CFLAGS)				\
 	$(SMARTCARD_CFLAGS)			\
+	$(GSTREAMER_0_10_CFLAGS)		\
 	$(GSTREAMER_1_0_CFLAGS)			\
 	$(SPICE_PROTOCOL_CFLAGS)		\
 	$(SSL_CFLAGS)				\
@@ -46,6 +47,7 @@  libserver_la_LIBADD =							\
 	$(PIXMAN_LIBS)							\
 	$(SASL_LIBS)							\
 	$(SLIRP_LIBS)							\
+	$(GSTREAMER_0_10_LIBS)						\
 	$(GSTREAMER_1_0_LIBS)						\
 	$(SSL_LIBS)							\
 	$(Z_LIBS)							\
@@ -159,6 +161,12 @@  libserver_la_SOURCES +=	\
 	$(NULL)
 endif
 
+if HAVE_GSTREAMER_0_10
+libserver_la_SOURCES +=	\
+	gstreamer-encoder.c			\
+	$(NULL)
+endif
+
 if HAVE_GSTREAMER_1_0
 libserver_la_SOURCES +=	\
 	gstreamer-encoder.c			\
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index b94656b..b7ab058 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -33,19 +33,28 @@ 
 
 #define SPICE_GST_DEFAULT_FPS 30
 
-#define DO_ZERO_COPY
+#ifndef HAVE_GSTREAMER_0_10
+# define DO_ZERO_COPY
+#endif
 
 
 typedef struct {
     SpiceBitmapFmt spice_format;
     const char *format;
     uint32_t bpp;
+    uint32_t depth;
+    uint32_t endianness;
+    uint32_t blue_mask;
+    uint32_t green_mask;
+    uint32_t red_mask;
 } SpiceFormatForGStreamer;
 
 typedef struct SpiceGstVideoBuffer {
     VideoBuffer base;
     GstBuffer *gst_buffer;
+#ifndef HAVE_GSTREAMER_0_10
     GstMapInfo map;
+#endif
 } SpiceGstVideoBuffer;
 
 typedef struct {
@@ -282,7 +291,9 @@  static void spice_gst_video_buffer_free(VideoBuffer *video_buffer)
 {
     SpiceGstVideoBuffer *buffer = (SpiceGstVideoBuffer*)video_buffer;
     if (buffer->gst_buffer) {
+#ifndef HAVE_GSTREAMER_0_10
         gst_buffer_unmap(buffer->gst_buffer, &buffer->map);
+#endif
         gst_buffer_unref(buffer->gst_buffer);
     }
     free(buffer);
@@ -742,11 +753,11 @@  static const SpiceFormatForGStreamer *map_format(SpiceBitmapFmt format)
      * section-types-definitions.html documents.
      */
     static const SpiceFormatForGStreamer format_map[] =  {
-        {SPICE_BITMAP_FMT_RGBA, "BGRA", 32},
-        {SPICE_BITMAP_FMT_16BIT, "RGB15", 16},
+        {SPICE_BITMAP_FMT_RGBA, "BGRA", 32, 24, 4321, 0xff000000, 0xff0000, 0xff00},
+        {SPICE_BITMAP_FMT_16BIT, "RGB15", 16, 15, 4321, 0x001f, 0x03E0, 0x7C00},
         /* TODO: Test the other formats */
-        {SPICE_BITMAP_FMT_32BIT, "BGRx", 32},
-        {SPICE_BITMAP_FMT_24BIT, "BGR", 24},
+        {SPICE_BITMAP_FMT_32BIT, "BGRx", 32, 24, 4321, 0xff000000, 0xff0000, 0xff00},
+        {SPICE_BITMAP_FMT_24BIT, "BGR", 24, 24, 4321, 0xff0000, 0xff00, 0xff},
     };
 
     int i;
@@ -768,8 +779,18 @@  static void set_appsrc_caps(SpiceGstEncoder *encoder)
         gst_caps_unref(encoder->src_caps);
     }
     encoder->src_caps = gst_caps_new_simple(
+#ifdef HAVE_GSTREAMER_0_10
+        "video/x-raw-rgb",
+        "bpp", G_TYPE_INT, encoder->format->bpp,
+        "depth", G_TYPE_INT, encoder->format->depth,
+        "endianness", G_TYPE_INT, encoder->format->endianness,
+        "red_mask", G_TYPE_INT, encoder->format->red_mask,
+        "green_mask", G_TYPE_INT, encoder->format->green_mask,
+        "blue_mask", G_TYPE_INT, encoder->format->blue_mask,
+#else
         "video/x-raw",
         "format", G_TYPE_STRING, encoder->format->format,
+#endif
         "width", G_TYPE_INT, encoder->width,
         "height", G_TYPE_INT, encoder->height,
         "framerate", GST_TYPE_FRACTION, get_source_fps(encoder), 1,
@@ -807,6 +828,13 @@  static GstFlowReturn new_sample(GstAppSink *gstappsink, gpointer video_encoder)
     SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
     SpiceGstVideoBuffer *outbuf = create_gst_video_buffer();
 
+#ifdef HAVE_GSTREAMER_0_10
+    outbuf->gst_buffer = gst_app_sink_pull_buffer(encoder->appsink);
+    if (outbuf->gst_buffer) {
+        outbuf->base.data = GST_BUFFER_DATA(outbuf->gst_buffer);
+        outbuf->base.size = GST_BUFFER_SIZE(outbuf->gst_buffer);
+    }
+#else
     GstSample *sample = gst_app_sink_pull_sample(encoder->appsink);
     if (sample) {
         outbuf->gst_buffer = gst_sample_get_buffer(sample);
@@ -817,6 +845,7 @@  static GstFlowReturn new_sample(GstAppSink *gstappsink, gpointer video_encoder)
             outbuf->base.size = gst_buffer_get_size(outbuf->gst_buffer);
         }
     }
+#endif
 
     /* Notify the main thread that the output buffer is ready */
     g_mutex_lock(&encoder->outbuf_mutex);
@@ -851,7 +880,11 @@  static const gchar* get_gst_codec_name(SpiceGstEncoder *encoder)
     switch (encoder->base.codec_type)
     {
     case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+#ifdef HAVE_GSTREAMER_0_10
+        return "ffenc_mjpeg";
+#else
         return "avenc_mjpeg";
+#endif
     case SPICE_VIDEO_CODEC_TYPE_VP8:
         return "vp8enc";
     case SPICE_VIDEO_CODEC_TYPE_H264:
@@ -865,6 +898,11 @@  static const gchar* get_gst_codec_name(SpiceGstEncoder *encoder)
 
 static gboolean create_pipeline(SpiceGstEncoder *encoder)
 {
+#ifdef HAVE_GSTREAMER_0_10
+    const gchar *converter = "ffmpegcolorspace";
+#else
+    const gchar *converter = "videoconvert";
+#endif
     const gchar* gstenc_name = get_gst_codec_name(encoder);
     if (!gstenc_name) {
         return FALSE;
@@ -873,12 +911,17 @@  static gboolean create_pipeline(SpiceGstEncoder *encoder)
     switch (encoder->base.codec_type)
     {
     case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+#ifdef HAVE_GSTREAMER_0_10
+        gstenc_opts = g_strdup("");
+#else
         /* Set max-threads to ensure zero-frame latency */
         gstenc_opts = g_strdup("max-threads=1");
+#endif
         break;
     case SPICE_VIDEO_CODEC_TYPE_VP8: {
         /* See http://www.webmproject.org/docs/encoder-parameters/
-         * - Set end-usage to get a constant bitrate to help with streaming.
+         * - Set mode/end-usage to get a constant bitrate to help with
+         *   streaming.
          * - min-quantizer ensures the bitrate does not get needlessly high.
          * - resize-allowed would be useful for low bitrate situations but
          *   the decoder does not return a frame of the expected size so
@@ -886,19 +929,23 @@  static gboolean create_pipeline(SpiceGstEncoder *encoder)
          * - error-resilient minimises artifacts in case the client drops a
          *   frame.
          * - Set lag-in-frames, deadline and cpu-used to match
-         *   "Profile Realtime". lag-in-frames ensures zero-frame latency,
-         *   deadline turns on realtime behavior, and cpu-used targets a 75%
-         *   CPU usage.
+         *   "Profile Realtime". max-latency/lag-in-frames ensures zero-frame
+         *   latency, deadline turns on realtime behavior, cpu-used targets a
+         *   75% CPU usage while speed simply prioritizes encoding speed.
          * - deadline is supposed to be set in microseconds but in practice
          *   it behaves like a boolean.
          * - At least up to GStreamer 1.6.2, vp8enc cannot be trusted to pick
          *   the optimal number of threads. Also exceeding the number of
          *   physical core really degrades image quality.
-         * - token-partitions parallelizes more operations.
+         * - token-parts/token-partitions parallelizes more operations.
          */
         int threads = get_physical_core_count();
         int parts = threads < 2 ? 0 : threads < 4 ? 1 : threads < 8 ? 2 : 3;
+#ifdef HAVE_GSTREAMER_0_10
+        gstenc_opts = g_strdup_printf("mode=cbr min-quantizer=10 error-resilient=true max-latency=0 speed=7 threads=%d token-parts=%d", threads, parts);
+#else
         gstenc_opts = g_strdup_printf("end-usage=cbr min-quantizer=10 error-resilient=default lag-in-frames=0 deadline=1 cpu-used=4 threads=%d token-partitions=%d", threads, parts);
+#endif
         break;
         }
     case SPICE_VIDEO_CODEC_TYPE_H264:
@@ -917,7 +964,7 @@  static gboolean create_pipeline(SpiceGstEncoder *encoder)
     }
 
     GError *err = NULL;
-    gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! videoconvert ! %s %s name=encoder ! appsink name=sink", gstenc_name, gstenc_opts);
+    gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! %s ! %s %s name=encoder ! appsink name=sink", converter, gstenc_name, gstenc_opts);
     spice_debug("GStreamer pipeline: %s", desc);
     encoder->pipeline = gst_parse_launch_full(desc, NULL, GST_PARSE_FLAG_FATAL_ERRORS, &err);
     g_free(gstenc_opts);
@@ -935,12 +982,20 @@  static gboolean create_pipeline(SpiceGstEncoder *encoder)
     encoder->gstenc = gst_bin_get_by_name(GST_BIN(encoder->pipeline), "encoder");
     encoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(encoder->pipeline), "sink"));
 
+#ifdef HAVE_GSTREAMER_0_10
+    GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, NULL, {NULL}};
+#else
     GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, {NULL}};
+#endif
     gst_app_sink_set_callbacks(encoder->appsink, &appsink_cbs, encoder, NULL);
 
     /* Hook into the bus so we can handle errors */
     GstBus *bus = gst_element_get_bus(encoder->pipeline);
+#ifdef HAVE_GSTREAMER_0_10
+    gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder);
+#else
     gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder, NULL);
+#endif
     gst_object_unref(bus);
 
     if (encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG) {
@@ -996,6 +1051,18 @@  static void set_gstenc_bitrate(SpiceGstEncoder *encoder)
         g_object_set(gobject, prop, (guint)gst_bit_rate, NULL);
         break;
         }
+    case G_TYPE_LONG: {
+        GParamSpecLong *range = G_PARAM_SPEC_LONG(param);
+        gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+        g_object_set(gobject, prop, (glong)gst_bit_rate, NULL);
+        break;
+        }
+    case G_TYPE_ULONG: {
+        GParamSpecULong *range = G_PARAM_SPEC_ULONG(param);
+        gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+        g_object_set(gobject, prop, (gulong)gst_bit_rate, NULL);
+        break;
+        }
     case G_TYPE_INT64: {
         GParamSpecInt64 *range = G_PARAM_SPEC_INT64(param);
         gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
@@ -1230,9 +1297,25 @@  static inline int chunk_copy(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap
     return TRUE;
 }
 
-/* A helper for push_raw_frame() */
+#ifdef HAVE_GSTREAMER_0_10
+/* Dummy structure to avoid too many #ifdef in the main codepaths */
+typedef struct {
+    gpointer memory;
+} GstMapInfo;
+#endif
+
+/* A helper for push_raw_frame()
+ * Note: In case of error the buffer is unref-ed.
+ */
 static uint8_t *allocate_and_map_memory(gsize size, GstMapInfo *map, GstBuffer *buffer)
 {
+#ifdef HAVE_GSTREAMER_0_10
+    buffer->malloc_data = g_malloc(size);
+    GST_BUFFER_DATA(buffer) = buffer->malloc_data;
+    GST_BUFFER_SIZE(buffer) = size;
+
+    return GST_BUFFER_DATA(buffer);
+#else
     GstMemory *mem = gst_allocator_alloc(NULL, size, NULL);
     if (!mem) {
         gst_buffer_unref(buffer);
@@ -1244,6 +1327,16 @@  static uint8_t *allocate_and_map_memory(gsize size, GstMapInfo *map, GstBuffer *
         return NULL;
     }
     return map->data;
+#endif
+}
+
+static void unmap_and_release_memory(GstMapInfo *map, GstBuffer *buffer)
+{
+#ifndef HAVE_GSTREAMER_0_10
+    gst_memory_unmap(map->memory, map);
+    gst_memory_unref(map->memory);
+#endif
+    gst_buffer_unref(buffer);
 }
 
 /* A helper for spice_gst_encoder_encode_frame() */
@@ -1276,9 +1369,7 @@  static int push_raw_frame(SpiceGstEncoder *encoder,
 
         chunk_offset += src->left * encoder->format->bpp / 8;
         if (!line_copy(encoder, bitmap, chunk_offset, stream_stride, height, dst)) {
-            gst_memory_unmap(map.memory, &map);
-            gst_memory_unref(map.memory);
-            gst_buffer_unref(buffer);
+            unmap_and_release_memory(&map, buffer);
             return VIDEO_ENCODER_FRAME_UNSUPPORTED;
         }
     } else {
@@ -1302,18 +1393,21 @@  static int push_raw_frame(SpiceGstEncoder *encoder,
             if (!dst) {
                 return VIDEO_ENCODER_FRAME_UNSUPPORTED;
             }
-            if (!chunk_copy(encoder, bitmap, chunk_index, chunk_offset, len, dst)) {
-                gst_memory_unmap(map.memory, &map);
-                gst_memory_unref(map.memory);
-                gst_buffer_unref(buffer);
+            if (!chunk_copy(encoder, bitmap, chunk_index, chunk_offset,
+                            len, dst)) {
+                unmap_and_release_memory(&map, buffer);
                 return VIDEO_ENCODER_FRAME_UNSUPPORTED;
             }
         }
     }
+#ifdef HAVE_GSTREAMER_0_10
+    gst_buffer_set_caps(buffer, encoder->src_caps);
+#else
     if (map.memory) {
         gst_memory_unmap(map.memory, &map);
         gst_buffer_append_memory(buffer, map.memory);
     }
+#endif
 
     GstFlowReturn ret = gst_app_src_push_buffer(encoder->appsrc, buffer);
     if (ret != GST_FLOW_OK) {
diff --git a/server/reds.c b/server/reds.c
index 0e320e8..f2d5434 100644
--- a/server/reds.c
+++ b/server/reds.c
@@ -3577,7 +3577,7 @@  static const EnumNames video_encoder_names[] = {
 
 static new_video_encoder_t video_encoder_procs[] = {
     &mjpeg_encoder_new,
-#ifdef HAVE_GSTREAMER_1_0
+#if defined(HAVE_GSTREAMER_1_0) || defined(HAVE_GSTREAMER_0_10)
     &gstreamer_encoder_new,
 #else
     NULL,
diff --git a/server/video-encoder.h b/server/video-encoder.h
index 3423118..5522818 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -197,7 +197,7 @@  VideoEncoder* mjpeg_encoder_new(SpiceVideoCodecType codec_type,
                                 VideoEncoderRateControlCbs *cbs,
                                 bitmap_ref_t bitmap_ref,
                                 bitmap_unref_t bitmap_unref);
-#ifdef HAVE_GSTREAMER_1_0
+#if defined(HAVE_GSTREAMER_1_0) || defined(HAVE_GSTREAMER_0_10)
 VideoEncoder* gstreamer_encoder_new(SpiceVideoCodecType codec_type,
                                     uint64_t starting_bit_rate,
                                     VideoEncoderRateControlCbs *cbs,