diff --git a/autogen.sh b/autogen.sh index 2c45ea9d..878cf054 100755 --- a/autogen.sh +++ b/autogen.sh @@ -6,27 +6,42 @@ srcdir=`dirname $0` (test -d $srcdir/m4) || mkdir $srcdir/m4 -pushd $srcdir > /dev/null -gtkdocize && \ -autoreconf --verbose --force --install --make || { - echo 'autogen.sh failed'; - exit 1; -} - -popd > /dev/null - -while test "x$@" != "x" ; do -optarg=`expr "x$@" : 'x[^=]*=\(.*\)'` -case "$@" in +for ag_option in $@ +do +case $ag_option in --noconfigure) NOCONFIGURE=defined AUTOGEN_EXT_OPT="$AUTOGEN_EXT_OPT --noconfigure" echo "+ configure run disabled" - shift + ;; + --disable-gtk-doc) + enable_gtk_doc=no + echo "+ gtk-doc disabled" ;; esac done +pushd $srcdir > /dev/null + +if test x$enable_gtk_doc = xno; then + if test -f gtk-doc.make; then :; else + echo "EXTRA_DIST = missing-gtk-doc" > gtk-doc.make + fi + echo "WARNING: You have disabled gtk-doc." + echo " As a result, you will not be able to generate the API" + echo " documentation and 'make dist' will not work." + echo +else + gtkdocize || exit $? +fi + +autoreconf --verbose --force --install --make || { + echo 'autogen.sh failed'; + exit 1; +} + +popd > /dev/null + for arg do CONFIGURE_EXT_OPT="$CONFIGURE_EXT_OPT $arg"; done if test ! -z "$CONFIGURE_EXT_OPT" then diff --git a/bridge/worker/peerhandler.js b/bridge/worker/peerhandler.js index 7a0d6b38..34fa53f2 100644 --- a/bridge/worker/peerhandler.js +++ b/bridge/worker/peerhandler.js @@ -62,6 +62,7 @@ function PeerHandler(configuration, keyCert, client, jsonRpc) { transportAgent.add_session(sessions[i]); } + transportAgent.start(); numberOfReceivePreparedSessions = sessions.length; function prepareSession(session, mdesc) { diff --git a/configure.ac b/configure.ac index b509d3f9..31eee080 100644 --- a/configure.ac +++ b/configure.ac @@ -28,7 +28,7 @@ AC_SUBST(LIBOPENWEBRTC_CFLAGS) GST_REQUIRED=1.4 PKG_CHECK_MODULES(GLIB, [glib-2.0, gobject-2.0, gmodule-2.0, gthread-2.0]) -PKG_CHECK_MODULES(GSTREAMER, [gstreamer-1.0 >= $GST_REQUIRED gstreamer-rtp-1.0 >= $GST_REQUIRED gstreamer-video-1.0 >= $GST_REQUIRED gstreamer-app-1.0 >= $GST_REQUIRED gstreamer-gl-1.0 >= $GST_REQUIRED]) +PKG_CHECK_MODULES(GSTREAMER, [gstreamer-1.0 >= $GST_REQUIRED gstreamer-rtp-1.0 >= $GST_REQUIRED gstreamer-video-1.0 >= $GST_REQUIRED gstreamer-audio-1.0 >= $GST_REQUIRED gstreamer-app-1.0 >= $GST_REQUIRED gstreamer-gl-1.0 >= $GST_REQUIRED]) PKG_CHECK_MODULES(NICE, [nice >= 0.1.7.1]) PKG_CHECK_MODULES(GSTREAMER_SCTP, [gstreamer-sctp-1.0]) PKG_CHECK_MODULES(ORC, [orc-0.4]) @@ -82,7 +82,8 @@ if test "x$enable_debug" = xyes; then fi AM_CONDITIONAL(OWR_DEBUG, test x$enable_debug = xyes) -dnl build debug or not +dnl build tests or not +have_json_glib=no AC_MSG_CHECKING([whether to build tests or not]) AC_ARG_ENABLE(tests, AC_HELP_STRING( @@ -95,12 +96,13 @@ AC_HELP_STRING( esac],[enable_tests=yes]) AC_MSG_RESULT([$enable_tests]) if test "x$enable_tests" = xyes; then - PKG_CHECK_MODULES(JSON_GLIB, [json-glib-1.0]) + PKG_CHECK_MODULES(JSON_GLIB, [json-glib-1.0], [$have_json_glib=yes], [$have_json_glib=no]) PKG_CHECK_MODULES(LIBSOUP, [libsoup-2.4]) AC_DEFINE(OWR_TESTS, 1, [Define if building tests]) fi AM_CONDITIONAL(OWR_TESTS, test x$enable_tests = xyes) +AM_CONDITIONAL(HAVE_JSON_GLIB, test x$have_json_glib = xyes) dnl build static or not AC_MSG_CHECKING([whether to build static owr or not]) @@ -184,6 +186,11 @@ esac AM_CONDITIONAL(TARGET_APPLE, test x$is_apple = xyes) dnl We substitute OWR_DEVICE_LIST_EXT_LIBS in the end +AC_CHECK_LIB([bcm_host], [bcm_host_init], +[AC_DEFINE(TARGET_RPI, 1, [Define if building for the Raspberry Pi])], +[AC_DEFINE(TARGET_RPI, 0, [Define if building for the Raspberry Pi])] +) + dnl generate java bindings or not AC_MSG_CHECKING([whether to generate java bindings or not]) AC_ARG_ENABLE(owr-java, diff --git a/libopenwebrtc.exp b/libopenwebrtc.exp index f6a3c72f..627ca640 100644 --- a/libopenwebrtc.exp +++ b/libopenwebrtc.exp @@ -69,6 +69,7 @@ owr_transport_agent_get_dot_data owr_transport_agent_get_type owr_transport_agent_new owr_transport_agent_set_local_port_range +owr_transport_agent_start owr_transport_type_get_type owr_uri_source_agent_get_dot_data owr_uri_source_agent_get_type diff --git a/local/owr_audio_renderer.c b/local/owr_audio_renderer.c index f80aef79..6690920b 100644 --- a/local/owr_audio_renderer.c +++ b/local/owr_audio_renderer.c @@ -55,6 +55,7 @@ GST_DEBUG_CATEGORY_EXTERN(_owraudiorenderer_debug); #define AUDIO_SINK "openslessink" #elif __linux__ +#include #define AUDIO_SINK "pulsesink" @@ -119,6 +120,28 @@ OwrAudioRenderer *owr_audio_renderer_new(void) NULL); } +static void +setup_sink_for_aec(GstElement *sink) +{ +#if defined(__linux__) && !defined(__ANDROID__) && !TARGET_RPI + /* pulsesink */ + GstStructure *s; + + s = gst_structure_new("props", PA_PROP_FILTER_WANT, G_TYPE_STRING, "echo-cancel", NULL); + g_object_set(G_OBJECT(sink), "stream-properties", s, NULL); + gst_structure_free(s); + +#elif defined(__ANDROID__) + /* openslessink */ + +#elif defined(__APPLE__) && !TARGET_IPHONE_SIMULATOR + /* osxaudiosink */ +#else + OWR_UNUSED(sink); +#endif +} + + #define LINK_ELEMENTS(a, b) \ if (!gst_element_link(a, b)) \ GST_ERROR("Failed to link " #a " -> " #b); @@ -151,6 +174,11 @@ static GstElement *owr_audio_renderer_get_element(OwrMediaRenderer *renderer) sink = OWR_MEDIA_RENDERER_GET_CLASS(renderer)->get_sink(renderer); g_assert(sink); + g_object_set(sink, "buffer-time", SINK_BUFFER_TIME, + "latency-time", G_GINT64_CONSTANT(10000), NULL); + + setup_sink_for_aec(sink); + gst_bin_add_many(GST_BIN(renderer_bin), audioresample, audioconvert, capsfilter, volume, sink, NULL); diff --git a/local/owr_device_list.c b/local/owr_device_list.c index 81453815..652a14c8 100644 --- a/local/owr_device_list.c +++ b/local/owr_device_list.c @@ -262,7 +262,8 @@ static void source_info_iterator(pa_context *pa_context, const pa_source_info *i } source = _owr_local_media_source_new_cached(info->index, info->description, - OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); context->list = g_list_prepend(context->list, source); } else { @@ -283,7 +284,8 @@ static gboolean enumerate_audio_source_devices(GClosure *callback) GList *sources = NULL; source = _owr_local_media_source_new_cached(-1, - "Default audio input", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE); + "Default audio input", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_prepend(sources, source); _owr_utils_call_closure_with_list(callback, sources); g_list_free_full(sources, g_object_unref); @@ -358,7 +360,8 @@ static OwrLocalMediaSource *maybe_create_source_from_filename(const gchar *name) return NULL; source = _owr_local_media_source_new_cached(index, device_name, - OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); g_debug("v4l: filename match: %s", device_name); @@ -378,6 +381,20 @@ static gboolean enumerate_video_source_devices(GClosure *callback) GDir *dev_dir; const gchar *filename; +#if TARGET_RPI + source = _owr_local_media_source_new_cached(-1, "RPiCam", + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION | OWR_MEDIA_SOURCE_SUPPORTS_COLOR_BALANCE); + _owr_media_source_set_codec(OWR_MEDIA_SOURCE(source), OWR_CODEC_TYPE_H264); + sources = g_list_prepend(sources, source); + + // Skip v4l2 devices. + _owr_utils_call_closure_with_list(callback, sources); + g_list_free_full(sources, g_object_unref); + + return FALSE; +#endif + dev_dir = g_dir_open("/dev", 0, &error); while ((filename = g_dir_read_name(dev_dir))) { @@ -681,11 +698,13 @@ static gboolean enumerate_video_source_devices(GClosure *callback) if (facing == CameraInfo.CAMERA_FACING_FRONT) { source = _owr_local_media_source_new_cached(i, "Front facing Camera", - OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_prepend(sources, source); } else if (facing == CameraInfo.CAMERA_FACING_BACK) { source = _owr_local_media_source_new_cached(i, "Back facing Camera", - OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_append(sources, source); } diff --git a/local/owr_local.c b/local/owr_local.c index 56b5bc97..de339d75 100644 --- a/local/owr_local.c +++ b/local/owr_local.c @@ -65,14 +65,17 @@ static GList *get_test_sources(OwrMediaType types) OwrMediaType media_type; GList *result_list = NULL; GList *elem; + gboolean useh264 = g_ascii_strcasecmp (g_getenv("OWR_USE_TEST_SOURCES"),"H264") == 0; if (g_once_init_enter(&cached_sources)) { GList *sources = NULL; - source = _owr_local_media_source_new_cached(-1, "Audio test source", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_TEST); + source = _owr_local_media_source_new_cached(-1, "Audio test source", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_TEST, OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_append(sources, OWR_MEDIA_SOURCE(source)); - source = _owr_local_media_source_new_cached(-1, "Video test source", OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_TEST); + source = _owr_local_media_source_new_cached(-1, "Video test source", OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_TEST, OWR_MEDIA_SOURCE_SUPPORTS_NONE); + if (useh264) + _owr_media_source_set_codec(OWR_MEDIA_SOURCE(source), OWR_CODEC_TYPE_H264); sources = g_list_append(sources, OWR_MEDIA_SOURCE(source)); g_once_init_leave(&cached_sources, sources); diff --git a/local/owr_local_media_source.c b/local/owr_local_media_source.c index b417fab5..14d596e3 100644 --- a/local/owr_local_media_source.c +++ b/local/owr_local_media_source.c @@ -45,6 +45,7 @@ #include "owr_utils.h" #include +#include #include #include @@ -65,9 +66,14 @@ GST_DEBUG_CATEGORY_EXTERN(_owrlocalmediasource_debug); #define VIDEO_SRC "androidvideosource" #elif defined(__linux__) +#include + #define AUDIO_SRC "pulsesrc" +#if TARGET_RPI +#define VIDEO_SRC "rpicamsrc" +#else #define VIDEO_SRC "v4l2src" - +#endif /* TARGET_RPI */ #else #define AUDIO_SRC "audiotestsrc" #define VIDEO_SRC "videotestsrc" @@ -78,6 +84,18 @@ static guint unique_bin_id = 0; #define OWR_LOCAL_MEDIA_SOURCE_GET_PRIVATE(obj) \ (G_TYPE_INSTANCE_GET_PRIVATE((obj), OWR_TYPE_LOCAL_MEDIA_SOURCE, OwrLocalMediaSourcePrivate)) +#define LINK_ELEMENTS(a, b) do { \ + if (!gst_element_link(a, b)) \ + GST_ERROR("Failed to link " #a " -> " #b); \ +} while (0) + +#define CREATE_ELEMENT(elem, factory, name) do { \ + elem = gst_element_factory_make(factory, name); \ + if (!elem) \ + GST_ERROR("Could not create " name " from factory " factory); \ + g_assert(elem); \ +} while (0) + static void owr_message_origin_interface_init(OwrMessageOriginInterface *interface); G_DEFINE_TYPE_WITH_CODE(OwrLocalMediaSource, owr_local_media_source, OWR_TYPE_MEDIA_SOURCE, @@ -86,6 +104,12 @@ G_DEFINE_TYPE_WITH_CODE(OwrLocalMediaSource, owr_local_media_source, OWR_TYPE_ME struct _OwrLocalMediaSourcePrivate { gint device_index; OwrMessageOriginBusSet *message_origin_bus_set; + + /* Volume control for audio sources */ + GstElement *source_volume; + /* Volume and mute are for before source_volume gets created */ + double volume; + gboolean mute; }; static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_source, GstCaps *caps); @@ -98,6 +122,8 @@ static void owr_local_media_source_get_property(GObject *object, guint property_ enum { PROP_0, PROP_DEVICE_INDEX, + PROP_VOLUME, + PROP_MUTE, N_PROPERTIES }; @@ -107,6 +133,8 @@ static void owr_local_media_source_finalize(GObject *object) owr_message_origin_bus_set_free(source->priv->message_origin_bus_set); source->priv->message_origin_bus_set = NULL; + + g_clear_object(&source->priv->source_volume); } static void owr_local_media_source_class_init(OwrLocalMediaSourceClass *klass) @@ -128,6 +156,17 @@ static void owr_local_media_source_class_init(OwrLocalMediaSourceClass *klass) "Index of the device to be used for this source (-1 => auto)", -1, G_MAXINT16, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property(gobject_class, PROP_VOLUME, + g_param_spec_double("volume", "Volume", + "Volume factor (only applicable to audio sources)", + 0, 1, 0.8, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property(gobject_class, PROP_MUTE, + g_param_spec_boolean("mute", "Mute", + "Mute state (only applicable to audio sources)", + FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } static gpointer owr_local_media_source_get_bus_set(OwrMessageOrigin *origin) @@ -143,21 +182,86 @@ static void owr_message_origin_interface_init(OwrMessageOriginInterface *interfa static void owr_local_media_source_init(OwrLocalMediaSource *source) { OwrLocalMediaSourcePrivate *priv; + source->priv = priv = OWR_LOCAL_MEDIA_SOURCE_GET_PRIVATE(source); priv->device_index = -1; priv->message_origin_bus_set = owr_message_origin_bus_set_new(); + priv->source_volume = NULL; + priv->volume = 0.8; + priv->mute = FALSE; } static void owr_local_media_source_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { OwrLocalMediaSource *source = OWR_LOCAL_MEDIA_SOURCE(object); + OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; switch (property_id) { case PROP_DEVICE_INDEX: source->priv->device_index = g_value_get_int(value); break; - + case PROP_VOLUME: { + gdouble volume_value = g_value_get_double(value); + + g_object_get(source, "media-type", &media_type, NULL); + + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + /* Set this anyway in case the source gets shutdown and restarted */ + source->priv->volume = volume_value; + + GST_DEBUG_OBJECT(source, "setting volume to %f\n", volume_value); + if (source->priv->source_volume) + g_object_set(source->priv->source_volume, "volume", volume_value, NULL); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + gst_stream_volume_set_volume(GST_STREAM_VOLUME(source_element), GST_STREAM_VOLUME_FORMAT_CUBIC, volume_value); + gst_object_unref(source_element); + } else + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + gst_object_unref(source_bin); + } else + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + } + } else + GST_WARNING_OBJECT(source, "Tried to set volume on non-audio source"); + break; + } + case PROP_MUTE: { + gboolean mute_value = g_value_get_boolean(value); + g_object_get(source, "media-type", &media_type, NULL); + + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + /* Set this anyway in case the source gets shutdown and restarted */ + source->priv->mute = mute_value; + + GST_DEBUG_OBJECT(source, "setting mute to %d\n", mute_value); + if (source->priv->source_volume) + g_object_set(source->priv->source_volume, "mute", mute_value, NULL); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + gst_stream_volume_set_mute(GST_STREAM_VOLUME(source_element), mute_value); + gst_object_unref(source_element); + } else + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + gst_object_unref(source_bin); + } else + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + } + } else + GST_WARNING_OBJECT(source, "Tried to set mute on non-audio source"); + break; + } default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -168,11 +272,72 @@ static void owr_local_media_source_get_property(GObject *object, guint property_ GValue *value, GParamSpec *pspec) { OwrLocalMediaSource *source = OWR_LOCAL_MEDIA_SOURCE(object); + OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; switch (property_id) { case PROP_DEVICE_INDEX: g_value_set_int(value, source->priv->device_index); break; + case PROP_VOLUME: + g_object_get(source, "media-type", &media_type, NULL); + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + if (source->priv->source_volume) + g_object_get_property(G_OBJECT(source->priv->source_volume), "volume", value); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + g_value_set_double(value, gst_stream_volume_get_volume(GST_STREAM_VOLUME(source_element), GST_STREAM_VOLUME_FORMAT_CUBIC)); + else + g_value_set_double(value, source->priv->volume); + + gst_object_unref(source_element); + } else { + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + g_value_set_double(value, source->priv->volume); + } + gst_object_unref(source_bin); + } else { + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + g_value_set_double(value, source->priv->volume); + } + } + } else + GST_WARNING_OBJECT(source, "Tried to get volume on non-audio source"); + break; + case PROP_MUTE: + g_object_get(source, "media-type", &media_type, NULL); + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + if (source->priv->source_volume) + g_object_get_property(G_OBJECT(source->priv->source_volume), "mute", value); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + g_value_set_boolean(value, gst_stream_volume_get_mute(GST_STREAM_VOLUME(source_element))); + else + g_value_set_boolean(value, source->priv->mute); + + gst_object_unref(source_element); + } else { + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + g_value_set_boolean(value, source->priv->mute); + } + gst_object_unref(source_bin); + } else { + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + g_value_set_boolean(value, source->priv->mute); + } + } + } else + GST_WARNING_OBJECT(source, "Tried to get volume on non-audio source"); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); @@ -180,16 +345,6 @@ static void owr_local_media_source_get_property(GObject *object, guint property_ } } -#define LINK_ELEMENTS(a, b) \ - if (!gst_element_link(a, b)) \ - GST_ERROR("Failed to link " #a " -> " #b); - -#define CREATE_ELEMENT(elem, factory, name) \ - elem = gst_element_factory_make(factory, name); \ - if (!elem) \ - GST_ERROR("Could not create " name " from factory " factory); \ - g_assert(elem); - /* FIXME: Copy from owr/orw.c without any error handling whatsoever */ static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer user_data) { @@ -266,6 +421,7 @@ static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer user_data) static gboolean shutdown_media_source(GHashTable *args) { OwrMediaSource *media_source; + OwrLocalMediaSource *local_media_source; GstElement *source_pipeline, *source_tee; GHashTable *event_data; GValue *value; @@ -277,6 +433,10 @@ static gboolean shutdown_media_source(GHashTable *args) media_source = g_hash_table_lookup(args, "media_source"); g_assert(media_source); + local_media_source = OWR_LOCAL_MEDIA_SOURCE(media_source); + if (local_media_source->priv->source_volume) + gst_object_unref(local_media_source->priv->source_volume); + source_pipeline = _owr_media_source_get_source_bin(media_source); if (!source_pipeline) { g_object_unref(media_source); @@ -406,7 +566,29 @@ static void on_caps(GstElement *source, GParamSpec *pspec, OwrMediaSource *media if (GST_IS_CAPS(caps)) { GST_INFO_OBJECT(source, "%s - configured with caps: %" GST_PTR_FORMAT, media_source_name, caps); + gst_caps_unref(caps); } + g_free(media_source_name); +} + +static void +setup_source_for_aec(GstElement *src) +{ +#if defined(__linux__) && !defined(__ANDROID__) + /* pulsesrc */ + GstStructure *s; + + s = gst_structure_new("props", PA_PROP_FILTER_WANT, G_TYPE_STRING, "echo-cancel", NULL); + g_object_set(G_OBJECT(src), "stream-properties", s, NULL); + gst_structure_free(s); + +#elif defined(__ANDROID__) + /* openslessrc */ + +#elif defined(__APPLE__) && !TARGET_IPHONE_SIMULATOR + /* osxaudiosrc */ + +#endif } /* @@ -451,6 +633,8 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s local_source = OWR_LOCAL_MEDIA_SOURCE(media_source); priv = local_source->priv; + GST_DEBUG_OBJECT(media_source, "source requested"); + /* only create the source bin for this media source once */ if ((source_pipeline = _owr_media_source_get_source_bin(media_source))) GST_DEBUG_OBJECT(media_source, "Re-using existing source element/bin"); @@ -537,6 +721,7 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s #endif } #endif + setup_source_for_aec(source); break; case OWR_SOURCE_TYPE_TEST: CREATE_ELEMENT(source, "audiotestsrc", "audio-source"); @@ -548,6 +733,13 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s goto done; } + if (!GST_IS_STREAM_VOLUME(source)) { + CREATE_ELEMENT(priv->source_volume, "volume", "audio-source-volume"); + g_object_set(priv->source_volume, "volume", priv->volume, "mute", priv->mute, NULL); + source_process = gst_object_ref(priv->source_volume); + gst_bin_add(GST_BIN(source_pipeline), source_process); + } + break; } case OWR_MEDIA_TYPE_VIDEO: @@ -558,6 +750,7 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s switch (source_type) { case OWR_SOURCE_TYPE_CAPTURE: CREATE_ELEMENT(source, VIDEO_SRC, "video-source"); +#if !defined(TARGET_RPI) || !TARGET_RPI if (priv->device_index > -1) { #if defined(__APPLE__) && !TARGET_IPHONE_SIMULATOR g_object_set(source, "device-index", priv->device_index, NULL); @@ -569,10 +762,19 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s g_free(tmp); #endif } +#else + g_object_set(source, "preview", FALSE, "fullscreen", FALSE, "inline-headers", TRUE, NULL); +#endif break; case OWR_SOURCE_TYPE_TEST: { - GstElement *src, *time; + GstElement *src, *time, *h264enc = NULL; GstPad *srcpad; + gboolean useh264 = g_ascii_strcasecmp (g_getenv("OWR_USE_TEST_SOURCES"),"H264") == 0; + + if (useh264) + printf("video-source encoding: video/x-h264\n"); + else + printf("video-source encoding: video/x-raw\n"); source = gst_bin_new("video-source"); @@ -585,9 +787,24 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s g_object_set(time, "font-desc", "Sans 60", NULL); gst_bin_add(GST_BIN(source), time); gst_element_link(src, time); - srcpad = gst_element_get_static_pad(time, "src"); - } else - srcpad = gst_element_get_static_pad(src, "src"); + if (!useh264) + srcpad = gst_element_get_static_pad(time, "src"); + } else if (!useh264) + srcpad = gst_element_get_static_pad(src, "src"); + + if (useh264) { + h264enc = gst_element_factory_make("openh264enc", "openh264enc"); + if (!h264enc) { + GST_ERROR_OBJECT(source, "Failed to create openh264enc element!"); + printf("Failed to create openh264enc element!\n"); + } + gst_bin_add(GST_BIN(source), h264enc); + if (time) + gst_element_link(time, h264enc); + else + gst_element_link(src, h264enc); + srcpad = gst_element_get_static_pad(h264enc, "src"); + } gst_element_add_pad(source, gst_ghost_pad_new("src", srcpad)); gst_object_unref(srcpad); @@ -644,6 +861,37 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s gst_caps_set_simple(source_caps, "format", G_TYPE_STRING, "NV12", NULL); #endif +#if defined(TARGET_RPI) && TARGET_RPI +#define MAX_RPI_CAMERA_HEIGHT 720 +#define MAX_RPI_CAMERA_WIDTH 1440 + /* The requested video source resolution can cause firmware mmal related errors. + * This happens for example with WebKit and AppRTC when the screen resolution is set to 1080p. + * We work-around this by capping the maximum allowed resolution to 720p. + * We try to keep the aspect ratio. */ + gint width, height; + gdouble ratio; + const GstStructure *str; + str = gst_caps_get_structure (source_caps, 0); + if (gst_structure_get_int (str, "width", &width) && + gst_structure_get_int (str, "height", &height)) { + GST_DEBUG_OBJECT(local_source, "RPiCam: Asked source video resolution of %dx%d\n", width, height); + if ((height > MAX_RPI_CAMERA_HEIGHT) || + (width > MAX_RPI_CAMERA_WIDTH)) { + ratio = (gdouble) width / height; + if (height > MAX_RPI_CAMERA_HEIGHT) { + height = MAX_RPI_CAMERA_HEIGHT; + width = height * ratio; + } + if (width > MAX_RPI_CAMERA_WIDTH) { + width = MAX_RPI_CAMERA_WIDTH; + height = width / ratio; + } + gst_caps_set_simple(source_caps, "height", G_TYPE_INT, height, NULL); + gst_caps_set_simple(source_caps, "width", G_TYPE_INT, width, NULL); + GST_WARNING_OBJECT(local_source, "RPiCam: The asked source video resolution was capped to %dx%d to avoid mmal firmware related errors\n", width, height); + } + } +#endif //defined(TARGET_RPI) && TARGET_RPI CREATE_ELEMENT(capsfilter, "capsfilter", "video-source-capsfilter"); g_object_set(capsfilter, "caps", source_caps, NULL); gst_caps_unref(source_caps); @@ -722,7 +970,8 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s } static OwrLocalMediaSource *_owr_local_media_source_new(gint device_index, const gchar *name, - OwrMediaType media_type, OwrSourceType source_type) + OwrMediaType media_type, OwrSourceType source_type, + OwrMediaSourceSupportedInterfaces interfaces) { OwrLocalMediaSource *source; @@ -733,12 +982,14 @@ static OwrLocalMediaSource *_owr_local_media_source_new(gint device_index, const NULL); _owr_media_source_set_type(OWR_MEDIA_SOURCE(source), source_type); + _owr_media_source_set_supported_interfaces(OWR_MEDIA_SOURCE(source), interfaces); return source; } OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, const gchar *name, - OwrMediaType media_type, OwrSourceType source_type) + OwrMediaType media_type, OwrSourceType source_type, + OwrMediaSourceSupportedInterfaces interfaces) { static OwrLocalMediaSource *test_sources[2] = { NULL, }; static GHashTable *sources[2] = { NULL, }; @@ -759,7 +1010,7 @@ OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, const if (source_type == OWR_SOURCE_TYPE_TEST) { if (!test_sources[i]) - test_sources[i] = _owr_local_media_source_new(device_index, name, media_type, source_type); + test_sources[i] = _owr_local_media_source_new(device_index, name, media_type, source_type, interfaces); ret = test_sources[i]; @@ -779,7 +1030,7 @@ OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, const } if (!ret) { - ret = _owr_local_media_source_new(device_index, name, media_type, source_type); + ret = _owr_local_media_source_new(device_index, name, media_type, source_type, interfaces); g_hash_table_insert(sources[i], GINT_TO_POINTER(device_index), ret); } diff --git a/local/owr_local_media_source_private.h b/local/owr_local_media_source_private.h index 34c36528..b0f1eba1 100644 --- a/local/owr_local_media_source_private.h +++ b/local/owr_local_media_source_private.h @@ -33,6 +33,7 @@ #define __OWR_LOCAL_MEDIA_SOURCE_PRIVATE_H__ #include "owr_local_media_source.h" +#include "owr_media_source_private.h" #include "owr_types.h" @@ -43,7 +44,8 @@ G_BEGIN_DECLS OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, - const gchar *name, OwrMediaType media_type, OwrSourceType source_type); + const gchar *name, OwrMediaType media_type, OwrSourceType source_type, + OwrMediaSourceSupportedInterfaces interfaces); void _owr_local_media_source_set_capture_device_index(OwrLocalMediaSource *source, guint index); G_END_DECLS diff --git a/local/owr_media_renderer.c b/local/owr_media_renderer.c index 5d688e8d..e62ee001 100644 --- a/local/owr_media_renderer.c +++ b/local/owr_media_renderer.c @@ -58,6 +58,7 @@ enum { PROP_0, PROP_MEDIA_TYPE, PROP_DISABLED, + PROP_SOURCE, N_PROPERTIES }; @@ -130,6 +131,10 @@ static void owr_media_renderer_class_init(OwrMediaRendererClass *klass) "Whether this renderer is disabled or not", DEFAULT_DISABLED, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_SOURCE] = g_param_spec_object("source", "Source", + "Current Media Source being rendered", OWR_TYPE_MEDIA_SOURCE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + gobject_class->set_property = owr_media_renderer_set_property; gobject_class->get_property = owr_media_renderer_get_property; @@ -263,6 +268,10 @@ static void owr_media_renderer_set_property(GObject *object, guint property_id, priv->disabled = g_value_get_boolean(value); break; + case PROP_SOURCE: + priv->source = g_value_get_object(value); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -286,6 +295,10 @@ static void owr_media_renderer_get_property(GObject *object, guint property_id, g_value_set_boolean(value, priv->disabled); break; + case PROP_SOURCE: + g_value_set_object(value, priv->source); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -305,6 +318,7 @@ static void on_caps(GstElement *sink, GParamSpec *pspec, OwrMediaRenderer *media media_renderer->priv->media_type == OWR_MEDIA_TYPE_AUDIO ? "Audio" : media_renderer->priv->media_type == OWR_MEDIA_TYPE_VIDEO ? "Video" : "Unknown", caps); + gst_caps_unref(caps); } } @@ -388,6 +402,7 @@ static gboolean set_source(GHashTable *args) } priv->source = g_object_ref(source); + g_object_notify_by_pspec(G_OBJECT(renderer), obj_properties[PROP_SOURCE]); maybe_start_renderer(renderer); @@ -469,6 +484,11 @@ void _owr_media_renderer_set_sink(OwrMediaRenderer *renderer, gpointer sink_ptr) g_mutex_unlock(&priv->media_renderer_lock); } +OwrMediaSource* _owr_media_renderer_get_source(OwrMediaRenderer *renderer) +{ + return renderer->priv->source; +} + gchar * owr_media_renderer_get_dot_data(OwrMediaRenderer *renderer) { g_return_val_if_fail(OWR_IS_MEDIA_RENDERER(renderer), NULL); diff --git a/local/owr_media_renderer.h b/local/owr_media_renderer.h index bb29a214..a67878a2 100644 --- a/local/owr_media_renderer.h +++ b/local/owr_media_renderer.h @@ -63,6 +63,7 @@ struct _OwrMediaRendererClass { /*< private >*/ void *(*get_caps)(OwrMediaRenderer *renderer); void *(*get_sink)(OwrMediaRenderer *renderer); + }; GType owr_media_renderer_get_type(void) G_GNUC_CONST; diff --git a/local/owr_media_renderer_private.h b/local/owr_media_renderer_private.h index dc99e22a..045491c7 100644 --- a/local/owr_media_renderer_private.h +++ b/local/owr_media_renderer_private.h @@ -39,6 +39,8 @@ G_BEGIN_DECLS void _owr_media_renderer_set_sink(OwrMediaRenderer *renderer, gpointer sink); GstPipeline * _owr_media_renderer_get_pipeline(OwrMediaRenderer *renderer); +OwrMediaSource* _owr_media_renderer_get_source(OwrMediaRenderer *renderer); +void _owr_media_renderer_reconfigure_element(OwrMediaRenderer *renderer); G_END_DECLS diff --git a/local/owr_video_renderer.c b/local/owr_video_renderer.c index 8f1ab142..5a2c092d 100644 --- a/local/owr_video_renderer.c +++ b/local/owr_video_renderer.c @@ -35,12 +35,14 @@ #include "owr_video_renderer.h" #include "owr_media_renderer_private.h" +#include "owr_media_source_private.h" #include "owr_private.h" #include "owr_utils.h" #include "owr_video_renderer_private.h" #include "owr_window_registry.h" #include "owr_window_registry_private.h" +#include #include #include @@ -86,9 +88,12 @@ static void owr_video_renderer_get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec); static void owr_video_renderer_constructed(GObject *object); -static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, guintptr window_handle); +static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer); +static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer); +static GstElement *owr_video_renderer_get_element_with_window_handle(OwrMediaRenderer *renderer, guintptr window_handle); static GstCaps *owr_video_renderer_get_caps(OwrMediaRenderer *renderer); static GstElement *owr_video_renderer_get_sink(OwrMediaRenderer *renderer); +static void _owr_video_renderer_notify_source_changed(OwrMediaRenderer *renderer, GParamSpec *pspec, gpointer user_data); struct _OwrVideoRendererPrivate { guint width; @@ -99,6 +104,7 @@ struct _OwrVideoRendererPrivate { gchar *tag; GMutex closure_mutex; GClosure *request_context; + GstElement *renderer_bin; }; static void owr_video_renderer_finalize(GObject *object) @@ -119,6 +125,11 @@ static void owr_video_renderer_finalize(GObject *object) priv->request_context = NULL; } + if (priv->renderer_bin) { + gst_object_unref(priv->renderer_bin); + priv->renderer_bin = NULL; + } + G_OBJECT_CLASS(owr_video_renderer_parent_class)->finalize(object); } @@ -162,6 +173,7 @@ static void owr_video_renderer_class_init(OwrVideoRendererClass *klass) media_renderer_class->get_caps = (void *(*)(OwrMediaRenderer *))owr_video_renderer_get_caps; media_renderer_class->get_sink = (void *(*)(OwrMediaRenderer *))owr_video_renderer_get_sink; + g_object_class_install_properties(gobject_class, N_PROPERTIES, obj_properties); } @@ -178,6 +190,9 @@ static void owr_video_renderer_init(OwrVideoRenderer *renderer) priv->mirror = DEFAULT_MIRROR; g_mutex_init(&priv->closure_mutex); priv->request_context = NULL; + priv->renderer_bin = NULL; + + g_signal_connect(renderer, "notify::source", G_CALLBACK(_owr_video_renderer_notify_source_changed), NULL); } static void owr_video_renderer_set_property(GObject *object, guint property_id, @@ -269,31 +284,67 @@ OwrVideoRenderer *owr_video_renderer_new(const gchar *tag) if (!gst_element_link(a, b)) \ GST_ERROR("Failed to link " #a " -> " #b); -static void renderer_disabled(OwrMediaRenderer *renderer, GParamSpec *pspec, GstElement *balance) +static void renderer_disabled(OwrMediaRenderer *renderer, G_GNUC_UNUSED GParamSpec *pspec, GstElement *balance) { + // FIXME: We need to be able to disable rendering without a + // balance element. This is highly inneficient. gboolean disabled = FALSE; + GstColorBalance* color_balance = NULL; g_return_if_fail(OWR_IS_MEDIA_RENDERER(renderer)); - g_return_if_fail(G_IS_PARAM_SPEC(pspec) || !pspec); - g_return_if_fail(GST_IS_ELEMENT(balance)); + + if (GST_IS_COLOR_BALANCE(balance)) { + color_balance = GST_COLOR_BALANCE(gst_object_ref(balance)); + } else { + OwrMediaSource* media_source = _owr_media_renderer_get_source(renderer); + GstElement* src_bin = _owr_media_source_get_source_bin(media_source); + balance = gst_bin_get_by_interface(GST_BIN(src_bin), GST_TYPE_COLOR_BALANCE); + gst_object_unref(src_bin); + g_return_if_fail(GST_IS_COLOR_BALANCE(balance)); + color_balance = GST_COLOR_BALANCE(balance); + } g_object_get(renderer, "disabled", &disabled, NULL); - g_object_set(balance, "saturation", (gdouble)!disabled, "brightness", (gdouble)-disabled, NULL); + + const GList* controls = gst_color_balance_list_channels(color_balance); + gint index = 0; + const GList* item; + for (item = controls; item != NULL; item = item->next, ++index) { + GstColorBalanceChannel* channel = item->data; + if (g_strcmp0(channel->label, "SATURATION") == 0 || g_strcmp0(channel->label, "BRIGHTNESS") == 0) { + gint new_value = disabled ? channel->min_value : ((channel->min_value + channel->max_value) / 2); + gst_color_balance_set_value(color_balance, channel, new_value); + } + } + + gst_object_unref(color_balance); } static void update_flip_method(OwrMediaRenderer *renderer, GParamSpec *pspec, GstElement *flip) { - guint rotation = 0; - gboolean mirror = FALSE; - gint flip_method; + g_assert(OWR_IS_MEDIA_RENDERER(renderer)); - g_return_if_fail(OWR_IS_MEDIA_RENDERER(renderer)); - g_return_if_fail(G_IS_PARAM_SPEC(pspec) || !pspec); - g_return_if_fail(GST_IS_ELEMENT(flip)); + if (!flip) { + OwrMediaSource* source = _owr_media_renderer_get_source(renderer); + + if (_owr_media_source_supports_interfaces(source, OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION)) { + GstElement* bin = _owr_media_source_get_source_bin(source); - g_object_get(renderer, "rotation", &rotation, "mirror", &mirror, NULL); - flip_method = _owr_rotation_and_mirror_to_video_flip_method(rotation, mirror); - g_object_set(flip, "method", flip_method, NULL); + flip = gst_bin_get_by_name(GST_BIN(bin), "video-source"); + g_assert(flip); + + pspec = g_object_class_find_property(G_OBJECT_GET_CLASS(flip), "orientation"); + + // For simplicity, and considering that we already assume + // with the parameter that the object is alive, we can + // safely unref here. + gst_object_unref(flip); + gst_object_unref(bin); + } + } + + if (flip) + _owr_update_flip_method(G_OBJECT(renderer), pspec, flip); } static void disable_last_sample_on_sink(const GValue *item, gpointer data) @@ -309,40 +360,89 @@ static void disable_last_sample_on_sink(const GValue *item, gpointer data) g_object_set(element, "enable-last-sample", FALSE, NULL); } -static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, guintptr window_handle) +static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer) { OwrVideoRenderer *video_renderer; OwrVideoRendererPrivate *priv; - GstElement *renderer_bin; - GstElement *upload, *convert, *balance, *flip, *sink; - GstPad *ghostpad, *sinkpad; gchar *bin_name; - GValue value = G_VALUE_INIT; - g_assert(renderer); + g_assert(OWR_IS_VIDEO_RENDERER(renderer)); video_renderer = OWR_VIDEO_RENDERER(renderer); priv = video_renderer->priv; bin_name = g_strdup_printf("video-renderer-bin-%u", g_atomic_int_add(&unique_bin_id, 1)); - renderer_bin = gst_bin_new(bin_name); + priv->renderer_bin = gst_bin_new(bin_name); g_free(bin_name); + return GST_ELEMENT(gst_object_ref(priv->renderer_bin)); +} + +static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer) +{ + OwrVideoRenderer *video_renderer; + OwrVideoRendererPrivate *priv; + GstElement *parser = NULL; + GstElement *decoder = NULL; + GstElement *balance = NULL; + GstElement *upload, *sink, *flip = NULL; + GstPad *ghostpad, *sinkpad; + GValue value = G_VALUE_INIT; + OwrMediaSource *source; + OwrCodecType codec_type; + gboolean link_ok = TRUE; + GstElement *first = NULL; + + g_assert(OWR_IS_VIDEO_RENDERER(renderer)); + video_renderer = OWR_VIDEO_RENDERER(renderer); + priv = video_renderer->priv; + + source = _owr_media_renderer_get_source(renderer); + codec_type = _owr_media_source_get_codec(source); + + if (!_owr_codec_type_is_raw(codec_type)) { + parser = _owr_create_parser(codec_type); + decoder = _owr_create_decoder(codec_type); + if (parser) + gst_bin_add(GST_BIN(priv->renderer_bin), parser); + if (decoder) + gst_bin_add(GST_BIN(priv->renderer_bin), decoder); + } + upload = gst_element_factory_make("glupload", "video-renderer-upload"); - convert = gst_element_factory_make("glcolorconvert", "video-renderer-convert"); + gst_bin_add(GST_BIN(priv->renderer_bin), upload); - balance = gst_element_factory_make("glcolorbalance", "video-renderer-balance"); - g_signal_connect_object(renderer, "notify::disabled", G_CALLBACK(renderer_disabled), - balance, 0); - renderer_disabled(renderer, NULL, balance); + if (!_owr_media_source_supports_interfaces(source, OWR_MEDIA_SOURCE_SUPPORTS_COLOR_BALANCE)) { + GstElement *convert = NULL; - flip = gst_element_factory_make("glvideoflip", "video-renderer-flip"); - if (!flip) { - g_warning("The glvideoflip GStreamer element isn't available. Video mirroring and rotation functionalities are thus disabled."); - } else { - g_signal_connect_object(renderer, "notify::rotation", G_CALLBACK(update_flip_method), flip, 0); - g_signal_connect_object(renderer, "notify::mirror", G_CALLBACK(update_flip_method), flip, 0); - update_flip_method(renderer, NULL, flip); + balance = gst_element_factory_make("glcolorbalance", "video-renderer-balance"); + + if (G_LIKELY(balance)) { + convert = gst_element_factory_make("glcolorconvert", "video-renderer-convert"); + + if (G_LIKELY(convert)) { + renderer_disabled(renderer, NULL, balance); + gst_bin_add_many(GST_BIN(priv->renderer_bin), convert, balance, NULL); + } else + g_object_unref(balance); + } + + if (!convert || !balance) + g_warning("cannot create convert or balance elements to disable rendering"); + } + g_signal_connect_object(renderer, "notify::disabled", G_CALLBACK(renderer_disabled), balance, 0); + + if (!_owr_media_source_supports_interfaces(source, OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION)) { + flip = gst_element_factory_make("glvideoflip", "video-renderer-flip"); + if (G_LIKELY(flip)) { + _owr_update_flip_method(G_OBJECT(renderer), NULL, flip); + gst_bin_add(GST_BIN(priv->renderer_bin), flip); + } else + g_warning("the glvideoflip element isn't available, video rotation support is now disabled"); } + g_signal_connect_object(renderer, "notify::rotation", G_CALLBACK(update_flip_method), flip, 0); + g_signal_connect_object(renderer, "notify::mirror", G_CALLBACK(update_flip_method), flip, 0); + + g_object_unref(source); sink = OWR_MEDIA_RENDERER_GET_CLASS(renderer)->get_sink(renderer); g_assert(sink); @@ -352,7 +452,31 @@ static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, gu disable_last_sample_on_sink(&value, NULL); g_value_unset(&value); - if (priv->tag) { + gst_bin_add(GST_BIN(priv->renderer_bin), sink); + + _owr_bin_link_and_sync_elements(GST_BIN(priv->renderer_bin), &link_ok, NULL, &first, NULL); + g_warn_if_fail(link_ok); + sinkpad = gst_element_get_static_pad(first, "sink"); + + g_assert(sinkpad); + ghostpad = gst_ghost_pad_new("sink", sinkpad); + gst_pad_set_active(ghostpad, TRUE); + gst_element_add_pad(priv->renderer_bin, ghostpad); + gst_object_unref(sinkpad); +} + +static GstElement *owr_video_renderer_get_element_with_window_handle(OwrMediaRenderer *renderer, guintptr window_handle) +{ + GstElement *renderer_bin, *sink; + + g_assert(OWR_IS_VIDEO_RENDERER(renderer)); + + renderer_bin = owr_video_renderer_get_element(renderer); + owr_video_renderer_reconfigure_element(renderer); + + sink = OWR_MEDIA_RENDERER_GET_CLASS(renderer)->get_sink(renderer); + g_assert(sink); + if (OWR_VIDEO_RENDERER(renderer)->priv->tag) { GstElement *sink_element = GST_IS_BIN(sink) ? gst_bin_get_by_interface(GST_BIN(sink), GST_TYPE_VIDEO_OVERLAY) : sink; if (GST_IS_ELEMENT(sink_element) && GST_IS_VIDEO_OVERLAY(sink)) @@ -363,26 +487,6 @@ static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, gu g_object_unref(sink_element); } - gst_bin_add_many(GST_BIN(renderer_bin), upload, convert, balance, sink, NULL); - - LINK_ELEMENTS(upload, convert); - LINK_ELEMENTS(convert, balance); - - if (flip) { - gst_bin_add(GST_BIN(renderer_bin), flip); - LINK_ELEMENTS(balance, flip); - LINK_ELEMENTS(flip, sink); - } else { - LINK_ELEMENTS(balance, sink); - } - - sinkpad = gst_element_get_static_pad(upload, "sink"); - g_assert(sinkpad); - ghostpad = gst_ghost_pad_new("sink", sinkpad); - gst_pad_set_active(ghostpad, TRUE); - gst_element_add_pad(renderer_bin, ghostpad); - gst_object_unref(sinkpad); - return renderer_bin; } @@ -452,7 +556,7 @@ static void owr_video_renderer_constructed(GObject *object) /* If we have no tag, just directly create the sink */ if (!priv->tag) - _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), owr_video_renderer_get_element(OWR_MEDIA_RENDERER(video_renderer), 0)); + _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), owr_video_renderer_get_element(OWR_MEDIA_RENDERER(video_renderer))); pipeline = _owr_media_renderer_get_pipeline(OWR_MEDIA_RENDERER(video_renderer)); g_assert(pipeline); @@ -469,6 +573,7 @@ static GstCaps *owr_video_renderer_get_caps(OwrMediaRenderer *renderer) GstCaps *caps; guint width = 0, height = 0; gdouble max_framerate = 0.0; + OwrMediaSource *source; g_object_get(OWR_VIDEO_RENDERER(renderer), "width", &width, @@ -476,7 +581,9 @@ static GstCaps *owr_video_renderer_get_caps(OwrMediaRenderer *renderer) "max-framerate", &max_framerate, NULL); - caps = gst_caps_new_empty_simple("video/x-raw"); + source = _owr_media_renderer_get_source(renderer); + caps = gst_caps_new_empty_simple(_owr_codec_type_to_caps_mime(_owr_media_source_get_media_type(source), + _owr_media_source_get_codec(source))); gst_caps_set_features(caps, 0, gst_caps_features_new_any()); if (width > 0) gst_caps_set_simple(caps, "width", G_TYPE_INT, width, NULL); @@ -498,6 +605,13 @@ static GstElement *owr_video_renderer_get_sink(OwrMediaRenderer *renderer) return gst_element_factory_make(VIDEO_SINK, "video-renderer-sink"); } +static void _owr_video_renderer_notify_source_changed(OwrMediaRenderer *renderer, GParamSpec *pspec, gpointer user_data) +{ + OWR_UNUSED(pspec); + OWR_UNUSED(user_data); + owr_video_renderer_reconfigure_element(renderer); +} + void _owr_video_renderer_notify_tag_changed(OwrVideoRenderer *video_renderer, const gchar *tag, gboolean have_handle, guintptr new_handle) { OwrVideoRendererPrivate *priv; @@ -511,6 +625,6 @@ void _owr_video_renderer_notify_tag_changed(OwrVideoRenderer *video_renderer, co _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), NULL); if (have_handle) { _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), - owr_video_renderer_get_element(OWR_MEDIA_RENDERER(video_renderer), new_handle)); + owr_video_renderer_get_element_with_window_handle(OWR_MEDIA_RENDERER(video_renderer), new_handle)); } } diff --git a/owr/owr.c b/owr/owr.c index eb705645..3f49c008 100644 --- a/owr/owr.c +++ b/owr/owr.c @@ -196,6 +196,8 @@ static void gst_log_android_handler(GstDebugCategory *category, */ void owr_init(GMainContext *main_context) { + static GOnce g_once = G_ONCE_INIT; + g_return_if_fail(!owr_initialized); #ifdef __ANDROID__ @@ -316,6 +318,8 @@ void owr_init(GMainContext *main_context) owr_main_context = g_main_context_ref_thread_default(); else g_main_context_ref(owr_main_context); + + g_once(&g_once, _owr_detect_codecs, NULL); } static gboolean owr_running_callback(GAsyncQueue *msg_queue) diff --git a/owr/owr_media_source.c b/owr/owr_media_source.c index 1580c466..8efb3dc1 100644 --- a/owr/owr_media_source.c +++ b/owr/owr_media_source.c @@ -121,6 +121,8 @@ struct _OwrMediaSourcePrivate { GstElement *source_bin; /* Tee element from which we can tap the source for multiple consumers */ GstElement *source_tee; + + OwrMediaSourceSupportedInterfaces supported_interfaces; }; static void owr_media_source_set_property(GObject *object, guint property_id, @@ -197,6 +199,7 @@ static void owr_media_source_init(OwrMediaSource *source) priv->source_bin = NULL; priv->source_tee = NULL; + priv->supported_interfaces = OWR_MEDIA_SOURCE_SUPPORTS_NONE; g_mutex_init(&source->lock); } @@ -263,7 +266,7 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media { OwrMediaType media_type; GstElement *source_pipeline, *tee; - GstElement *source_bin, *source = NULL, *queue_pre, *queue_post; + GstElement *source_bin, *source = NULL, *queue_pre = NULL, *queue_post = NULL; GstElement *capsfilter; GstElement *sink, *sink_queue, *sink_bin; GstPad *bin_pad = NULL, *srcpad, *sinkpad; @@ -283,9 +286,6 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media source_bin = gst_bin_new(bin_name); g_free(bin_name); - CREATE_ELEMENT_WITH_ID(queue_pre, "queue", "source-queue", source_id); - CREATE_ELEMENT_WITH_ID(capsfilter, "capsfilter", "source-output-capsfilter", source_id); - CREATE_ELEMENT_WITH_ID(queue_post, "queue", "source-output-queue", source_id); CREATE_ELEMENT_WITH_ID(sink_queue, "queue", "sink-queue", source_id); @@ -295,8 +295,12 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media { GstElement *audioresample, *audioconvert; + CREATE_ELEMENT_WITH_ID(capsfilter, "capsfilter", "source-output-capsfilter", source_id); g_object_set(capsfilter, "caps", caps, NULL); + CREATE_ELEMENT_WITH_ID(queue_pre, "queue", "source-queue", source_id); + CREATE_ELEMENT_WITH_ID(queue_post, "queue", "source-output-queue", source_id); + CREATE_ELEMENT_WITH_ID(audioresample, "audioresample", "source-audio-resample", source_id); CREATE_ELEMENT_WITH_ID(audioconvert, "audioconvert", "source-audio-convert", source_id); @@ -311,61 +315,71 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media } case OWR_MEDIA_TYPE_VIDEO: { - GstElement *videorate = NULL, *videoscale = NULL, *videoconvert; - GstStructure *s; - GstCapsFeatures *features; - - s = gst_caps_get_structure(caps, 0); - if (gst_structure_has_field(s, "framerate")) { - gint fps_n = 0, fps_d = 0; + if (_owr_codec_type_is_raw(_owr_media_source_get_codec(media_source))) { + GstElement *videorate = NULL, *videoscale = NULL, *videoconvert; + GstStructure *s; + GstCapsFeatures *features; - gst_structure_get_fraction(s, "framerate", &fps_n, &fps_d); - g_assert(fps_d); + CREATE_ELEMENT_WITH_ID(queue_pre, "queue", "source-queue", source_id); + CREATE_ELEMENT_WITH_ID(queue_post, "queue", "source-output-queue", source_id); - CREATE_ELEMENT_WITH_ID(videorate, "videorate", "source-video-rate", source_id); - g_object_set(videorate, "drop-only", TRUE, "max-rate", fps_n / fps_d, NULL); - - gst_structure_remove_field(s, "framerate"); - gst_bin_add(GST_BIN(source_bin), videorate); - } - g_object_set(capsfilter, "caps", caps, NULL); + s = gst_caps_get_structure(caps, 0); + if (gst_structure_has_field(s, "framerate")) { + gint fps_n = 0, fps_d = 0; - features = gst_caps_get_features(caps, 0); - if (gst_caps_features_contains(features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { - GstElement *glupload; + gst_structure_get_fraction(s, "framerate", &fps_n, &fps_d); + g_assert(fps_d); - CREATE_ELEMENT_WITH_ID(glupload, "glupload", "source-glupload", source_id); - CREATE_ELEMENT_WITH_ID(videoconvert, "glcolorconvert", "source-glcolorconvert", source_id); - gst_bin_add_many(GST_BIN(source_bin), - queue_pre, glupload, videoconvert, capsfilter, queue_post, NULL); + CREATE_ELEMENT_WITH_ID(videorate, "videorate", "source-video-rate", source_id); + g_object_set(videorate, "drop-only", TRUE, "max-rate", fps_n / fps_d, NULL); - if (videorate) { - LINK_ELEMENTS(queue_pre, videorate); - LINK_ELEMENTS(videorate, glupload); - } else { - LINK_ELEMENTS(queue_pre, glupload); + gst_structure_remove_field(s, "framerate"); + gst_bin_add(GST_BIN(source_bin), videorate); } - LINK_ELEMENTS(glupload, videoconvert); - } else { - GstElement *gldownload; - - CREATE_ELEMENT_WITH_ID(gldownload, "gldownload", "source-gldownload", source_id); - CREATE_ELEMENT_WITH_ID(videoscale, "videoscale", "source-video-scale", source_id); - CREATE_ELEMENT_WITH_ID(videoconvert, VIDEO_CONVERT, "source-video-convert", source_id); - gst_bin_add_many(GST_BIN(source_bin), - queue_pre, gldownload, videoscale, videoconvert, capsfilter, queue_post, NULL); - if (videorate) { - LINK_ELEMENTS(queue_pre, videorate); - LINK_ELEMENTS(videorate, gldownload); + + features = gst_caps_get_features(caps, 0); + if (gst_caps_features_contains(features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { + GstElement *glupload; + + CREATE_ELEMENT_WITH_ID(glupload, "glupload", "source-glupload", source_id); + CREATE_ELEMENT_WITH_ID(videoscale, "gleffects_identity", "source-glcolorscale", source_id); + CREATE_ELEMENT_WITH_ID(videoconvert, "glcolorconvert", "source-glcolorconvert", source_id); + + gst_bin_add_many(GST_BIN(source_bin), + queue_pre, glupload, videoconvert, videoscale, queue_post, NULL); + + if (videorate) { + LINK_ELEMENTS(queue_pre, videorate); + LINK_ELEMENTS(videorate, glupload); + } else { + LINK_ELEMENTS(queue_pre, glupload); + } + LINK_ELEMENTS(glupload, videoconvert); + LINK_ELEMENTS(videoconvert, videoscale); + LINK_ELEMENTS(videoscale, queue_post); } else { - LINK_ELEMENTS(queue_pre, gldownload); + GstElement *gldownload; + + CREATE_ELEMENT_WITH_ID(capsfilter, "capsfilter", "source-output-capsfilter", source_id); + g_object_set(capsfilter, "caps", caps, NULL); + + CREATE_ELEMENT_WITH_ID(gldownload, "gldownload", "source-gldownload", source_id); + CREATE_ELEMENT_WITH_ID(videoscale, "videoscale", "source-video-scale", source_id); + CREATE_ELEMENT_WITH_ID(videoconvert, VIDEO_CONVERT, "source-video-convert", source_id); + gst_bin_add_many(GST_BIN(source_bin), + queue_pre, gldownload, videoscale, videoconvert, capsfilter, queue_post, NULL); + if (videorate) { + LINK_ELEMENTS(queue_pre, videorate); + LINK_ELEMENTS(videorate, gldownload); + } else { + LINK_ELEMENTS(queue_pre, gldownload); + } + LINK_ELEMENTS(gldownload, videoscale); + LINK_ELEMENTS(videoscale, videoconvert); + LINK_ELEMENTS(videoconvert, capsfilter); + LINK_ELEMENTS(capsfilter, queue_post); } - LINK_ELEMENTS(gldownload, videoscale); - LINK_ELEMENTS(videoscale, videoconvert); } - LINK_ELEMENTS(videoconvert, capsfilter); - LINK_ELEMENTS(capsfilter, queue_post); - break; } case OWR_MEDIA_TYPE_UNKNOWN: @@ -404,7 +418,12 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media LINK_ELEMENTS(tee, sink_bin); /* Start up our new bin and link it all */ - srcpad = gst_element_get_static_pad(queue_post, "src"); + gst_bin_add(GST_BIN(source_bin), source); + if (queue_post) { + srcpad = gst_element_get_static_pad(queue_post, "src"); + } else { + srcpad = gst_element_get_static_pad(source, "src"); + } g_assert(srcpad); bin_pad = gst_ghost_pad_new("src", srcpad); @@ -412,14 +431,15 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media gst_pad_set_active(bin_pad, TRUE); gst_element_add_pad(source_bin, bin_pad); - gst_bin_add(GST_BIN(source_bin), source); - LINK_ELEMENTS(source, queue_pre); + if (queue_pre) + LINK_ELEMENTS(source, queue_pre); done: gst_object_unref(source_pipeline); gst_object_unref(tee); + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(source_bin), GST_DEBUG_GRAPH_SHOW_ALL, GST_OBJECT_NAME(GST_OBJECT(source_bin))); return source_bin; } @@ -615,6 +635,12 @@ void _owr_media_source_set_type(OwrMediaSource *media_source, OwrSourceType type g_atomic_int_set(&media_source->priv->type, type); } +OwrSourceType _owr_media_source_get_media_type(OwrMediaSource *media_source) +{ + g_return_val_if_fail(OWR_IS_MEDIA_SOURCE(media_source), OWR_MEDIA_TYPE_UNKNOWN); + return media_source->priv->media_type; +} + OwrCodecType _owr_media_source_get_codec(OwrMediaSource *media_source) { g_return_val_if_fail(OWR_IS_MEDIA_SOURCE(media_source), OWR_CODEC_TYPE_NONE); @@ -641,3 +667,15 @@ gchar * owr_media_source_get_dot_data(OwrMediaSource *source) return g_strdup(""); #endif } + +void _owr_media_source_set_supported_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces) +{ + g_return_if_fail(OWR_IS_MEDIA_SOURCE(source)); + source->priv->supported_interfaces = interfaces; +} + +gboolean _owr_media_source_supports_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces) +{ + g_return_val_if_fail(OWR_IS_MEDIA_SOURCE(source), FALSE); + return source->priv->supported_interfaces & interfaces; +} diff --git a/owr/owr_media_source_private.h b/owr/owr_media_source_private.h index ca9bb4ae..8aeab425 100644 --- a/owr/owr_media_source_private.h +++ b/owr/owr_media_source_private.h @@ -43,6 +43,12 @@ G_BEGIN_DECLS +typedef enum { + OWR_MEDIA_SOURCE_SUPPORTS_NONE = 0, + OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION = (1 << 0), + OWR_MEDIA_SOURCE_SUPPORTS_COLOR_BALANCE = (1 << 1), +} OwrMediaSourceSupportedInterfaces; + GstElement *_owr_media_source_get_source_bin(OwrMediaSource *media_source); void _owr_media_source_set_source_bin(OwrMediaSource *media_source, GstElement *bin); @@ -52,11 +58,19 @@ void _owr_media_source_set_source_tee(OwrMediaSource *media_source, GstElement * GstElement *_owr_media_source_request_source(OwrMediaSource *media_source, GstCaps *caps); void _owr_media_source_release_source(OwrMediaSource *media_source, GstElement *source); +/* FIXME: At some point we should rename this function to + * set_media_type because get_type could eventually conflict with + * GObject required function if this function becomes public. */ void _owr_media_source_set_type(OwrMediaSource *source, OwrSourceType type); +OwrSourceType _owr_media_source_get_media_type(OwrMediaSource *source); void _owr_media_source_set_codec(OwrMediaSource *source, OwrCodecType codec_type); OwrCodecType _owr_media_source_get_codec(OwrMediaSource *source); +void _owr_media_source_set_supported_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces); + +gboolean _owr_media_source_supports_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces); + G_END_DECLS #endif /* __GTK_DOC_IGNORE__ */ diff --git a/owr/owr_types.c b/owr/owr_types.c index 082291bb..8e3d2ddd 100644 --- a/owr/owr_types.c +++ b/owr/owr_types.c @@ -43,6 +43,7 @@ GType owr_codec_type_get_type(void) {OWR_CODEC_TYPE_OPUS, "Opus", "opus"}, {OWR_CODEC_TYPE_H264, "H264", "h264"}, {OWR_CODEC_TYPE_VP8, "VP8", "vp8"}, + {OWR_CODEC_TYPE_VP9, "VP9", "vp9"}, {0, NULL, NULL} }; static volatile GType id = 0; @@ -107,3 +108,21 @@ if (g_once_init_enter((gsize *)&id)) { return id; } + +GType owr_bundle_policy_type_get_type(void) +{ + static const GEnumValue types[] = { + {OWR_BUNDLE_POLICY_TYPE_BALANCED, "Pick two tracks to send, audio and video", "balanced"}, + {OWR_BUNDLE_POLICY_TYPE_MAX_COMPAT, "Separate each track into its own connection", "max_compat"}, + {OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE, "Pick one media track to negotiate and send just that one", "max_bundle"}, + {0, NULL, NULL} + }; +static volatile GType id = 0; + +if (g_once_init_enter((gsize *)&id)) { + GType _id = g_enum_register_static("OwrBundlePolicyTypes", types); + g_once_init_leave((gsize *)&id, _id); +} + +return id; +} diff --git a/owr/owr_types.h b/owr/owr_types.h index 762ca06c..a9c8ea32 100644 --- a/owr/owr_types.h +++ b/owr/owr_types.h @@ -42,7 +42,8 @@ typedef enum _OwrCodecType { OWR_CODEC_TYPE_PCMA, OWR_CODEC_TYPE_OPUS, OWR_CODEC_TYPE_H264, - OWR_CODEC_TYPE_VP8 + OWR_CODEC_TYPE_VP8, + OWR_CODEC_TYPE_VP9 } OwrCodecType; typedef enum _OwrMediaType { @@ -62,6 +63,12 @@ typedef enum _OwrAdaptationType { OWR_ADAPTATION_TYPE_SCREAM } OwrAdaptationType; +typedef enum _OwrBundlePolicyType { + OWR_BUNDLE_POLICY_TYPE_BALANCED, + OWR_BUNDLE_POLICY_TYPE_MAX_COMPAT, + OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE +} OwrBundlePolicyType; + #define OWR_TYPE_CODEC_TYPE (owr_codec_type_get_type()) GType owr_codec_type_get_type(void); @@ -74,6 +81,9 @@ GType owr_media_type_get_type(void); #define OWR_TYPE_ADAPTATION_TYPE (owr_adaptation_type_get_type()) GType owr_adaptation_type_get_type(void); +#define OWR_TYPE_BUNDLE_POLICY_TYPE (owr_bundle_policy_type_get_type()) +GType owr_bundle_policy_type_get_type(void); + G_END_DECLS diff --git a/owr/owr_utils.c b/owr/owr_utils.c index 6f41b736..e0228e8b 100644 --- a/owr/owr_utils.c +++ b/owr/owr_utils.c @@ -36,6 +36,25 @@ #include "owr_types.h" +/* To be extended once more codecs are supported */ +static GList *h264_decoders = NULL; +static GList *h264_encoders = NULL; +static GList *vp8_decoders = NULL; +static GList *vp8_encoders = NULL; +static GList *vp9_decoders = NULL; +static GList *vp9_encoders = NULL; + +static const gchar *OwrCodecTypeEncoderElementName[] = { NULL, "mulawenc", "alawenc", "opusenc", "openh264enc", "vp8enc", "vp9enc" }; +static const gchar *OwrCodecTypeDecoderElementName[] = { NULL, "mulawdec", "alawdec", "opusdec", "openh264dec", "vp8dec", "vp9dec" }; + +static const gchar *OwrCodecTypeParserElementName[] = { NULL, NULL, NULL, NULL, "h264parse", NULL, NULL }; + +guint _owr_get_unique_uint_id() +{ + static guint id = 0; + return g_atomic_int_add(&id, 1); +} + OwrCodecType _owr_caps_to_codec_type(GstCaps *caps) { GstStructure *structure; @@ -54,11 +73,234 @@ OwrCodecType _owr_caps_to_codec_type(GstCaps *caps) return OWR_CODEC_TYPE_H264; if (gst_structure_has_name(structure, "video/x-vp8")) return OWR_CODEC_TYPE_VP8; + if (gst_structure_has_name(structure, "video/x-vp9")) + return OWR_CODEC_TYPE_VP9; GST_ERROR("Unknown caps: %" GST_PTR_FORMAT, (gpointer)caps); return OWR_CODEC_TYPE_NONE; } +const gchar* _owr_codec_type_to_caps_mime(OwrMediaType media_type, OwrCodecType codec_type) +{ + switch (codec_type) + { + case OWR_CODEC_TYPE_NONE: + switch (media_type) + { + case OWR_MEDIA_TYPE_AUDIO: + return "audio/x-raw"; + break; + case OWR_MEDIA_TYPE_VIDEO: + return "video/x-raw"; + break; + default: + g_return_val_if_reached("audio/x-raw"); + } + break; + case OWR_CODEC_TYPE_PCMU: + return "audio/x-mulaw"; + break; + case OWR_CODEC_TYPE_PCMA: + return "audio/x-alaw"; + break; + case OWR_CODEC_TYPE_OPUS: + return "audio/x-opus"; + break; + case OWR_CODEC_TYPE_H264: + return "video/x-h264"; + break; + case OWR_CODEC_TYPE_VP8: + return "video/x-vp8"; + break; + case OWR_CODEC_TYPE_VP9: + return "video/x-vp9"; + break; + default: + break; + } + g_return_val_if_reached("audio/x-raw"); +} + +gpointer _owr_detect_codecs(gpointer data) +{ + GList *decoder_factories; + GList *encoder_factories; + GstCaps *caps; + + OWR_UNUSED(data); + + decoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | + GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, + GST_RANK_MARGINAL); + encoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER | + GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, + GST_RANK_MARGINAL); + + caps = gst_caps_new_empty_simple("video/x-h264"); + h264_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); + h264_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); + gst_caps_unref(caps); + + caps = gst_caps_new_empty_simple("video/x-vp8"); + vp8_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); + vp8_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); + gst_caps_unref(caps); + + caps = gst_caps_new_empty_simple("video/x-vp9"); + vp9_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); + vp9_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); + gst_caps_unref(caps); + + gst_plugin_feature_list_free(decoder_factories); + gst_plugin_feature_list_free(encoder_factories); + + h264_decoders = g_list_sort(h264_decoders, gst_plugin_feature_rank_compare_func); + h264_encoders = g_list_sort(h264_encoders, gst_plugin_feature_rank_compare_func); + vp8_decoders = g_list_sort(vp8_decoders, gst_plugin_feature_rank_compare_func); + vp8_encoders = g_list_sort(vp8_encoders, gst_plugin_feature_rank_compare_func); + vp9_decoders = g_list_sort(vp9_decoders, gst_plugin_feature_rank_compare_func); + vp9_encoders = g_list_sort(vp9_encoders, gst_plugin_feature_rank_compare_func); + + return NULL; +} + +const GList *_owr_get_detected_h264_encoders() +{ + return h264_encoders; +} + +const GList *_owr_get_detected_vp8_encoders() +{ + return vp8_encoders; +} + +const GList *_owr_get_detected_vp9_encoders() +{ + return vp9_encoders; +} + +GstElement *_owr_try_codecs(const GList *codecs, const gchar *name_prefix) +{ + GList *l; + gchar *element_name; + + for (l = (GList*) codecs; l; l = l->next) { + GstElementFactory *f = l->data; + GstElement *e; + + element_name = g_strdup_printf("%s_%s_%u", name_prefix, + gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(f)), + _owr_get_unique_uint_id()); + + e = gst_element_factory_create(f, element_name); + g_free(element_name); + + if (!e) + continue; + + /* Try setting to READY. If this fails the codec does not work, for + * example because the hardware codec is currently busy + */ + if (gst_element_set_state(e, GST_STATE_READY) != GST_STATE_CHANGE_SUCCESS) { + gst_element_set_state(e, GST_STATE_NULL); + gst_object_unref(e); + continue; + } + + return e; + } + + return NULL; +} + +GstElement * _owr_create_decoder(OwrCodecType codec_type) +{ + GstElement * decoder = NULL; + gchar *element_name = NULL; + + switch (codec_type) { + case OWR_CODEC_TYPE_H264: + decoder = _owr_try_codecs(h264_decoders, "decoder"); + g_return_val_if_fail(decoder, NULL); + break; + case OWR_CODEC_TYPE_VP8: + decoder = _owr_try_codecs(vp8_decoders, "decoder"); + g_return_val_if_fail(decoder, NULL); + break; + case OWR_CODEC_TYPE_VP9: + decoder = _owr_try_codecs(vp9_decoders, "decoder"); + g_return_val_if_fail(decoder, NULL); + break; + default: + element_name = g_strdup_printf("decoder_%s_%u", OwrCodecTypeDecoderElementName[codec_type], _owr_get_unique_uint_id()); + decoder = gst_element_factory_make(OwrCodecTypeDecoderElementName[codec_type], element_name); + g_free(element_name); + g_return_val_if_fail(decoder, NULL); + break; + } + + return decoder; +} + +GstElement * _owr_create_parser(OwrCodecType codec_type) +{ + GstElement * parser = NULL; + gchar *element_name = NULL; + + if (!OwrCodecTypeParserElementName[codec_type]) + return NULL; + + element_name = g_strdup_printf("parser_%s_%u", OwrCodecTypeParserElementName[codec_type], _owr_get_unique_uint_id()); + parser = gst_element_factory_make(OwrCodecTypeParserElementName[codec_type], element_name); + g_free(element_name); + + switch (codec_type) { + case OWR_CODEC_TYPE_H264: + g_object_set(parser, "disable-passthrough", TRUE, NULL); + break; + default: + break; + } + return parser; +} + +const gchar* _owr_get_encoder_name(OwrCodecType codec_type) +{ + return OwrCodecTypeEncoderElementName[codec_type]; +} + +void _owr_bin_link_and_sync_elements(GstBin *bin, gboolean *out_link_ok, gboolean *out_sync_ok, GstElement **out_first, GstElement **out_last) +{ + GList *bin_elements, *current; + gboolean link_ok = TRUE, sync_ok = TRUE; + + g_assert(bin); + + bin_elements = g_list_last(bin->children); + g_assert(bin_elements); + for (current = bin_elements; current && current->prev && link_ok && sync_ok; current = g_list_previous(current)) { + if (out_link_ok) + link_ok &= gst_element_link(current->data, current->prev->data); + if (out_sync_ok) + sync_ok &= gst_element_sync_state_with_parent(current->data); + } + if (out_sync_ok && link_ok && sync_ok && current && !current->prev) + sync_ok &= gst_element_sync_state_with_parent(current->data); + + if (out_link_ok) + *out_link_ok = link_ok; + + if (out_sync_ok) + *out_sync_ok = sync_ok; + + if (link_ok && sync_ok) { + if (out_first) + *out_first = bin_elements->data; + if (out_last) + *out_last = current->data; + } +} + typedef struct { GClosure *callback; GList *list; @@ -220,6 +462,19 @@ int _owr_rotation_and_mirror_to_video_flip_method(guint rotation, gboolean mirro } } +void _owr_update_flip_method(GObject *source, GParamSpec *pspec, GstElement *flip) +{ + guint rotation = 0; + gboolean mirror = FALSE; + gint flip_method; + + g_assert(GST_IS_ELEMENT(flip)); + + g_object_get(source, "rotation", &rotation, "mirror", &mirror, NULL); + flip_method = _owr_rotation_and_mirror_to_video_flip_method(rotation, mirror); + g_object_set(flip, pspec ? pspec->name : "method", flip_method, NULL); +} + static void value_slice_free(gpointer value) { g_value_unset(value); diff --git a/owr/owr_utils.h b/owr/owr_utils.h index 85e2754e..ae1d9109 100644 --- a/owr/owr_utils.h +++ b/owr/owr_utils.h @@ -39,8 +39,21 @@ G_BEGIN_DECLS #define OWR_UNUSED(x) (void)x +#define _owr_codec_type_is_raw(codec_type) (codec_type == OWR_CODEC_TYPE_NONE) + void *_owr_require_symbols(void); +guint _owr_get_unique_uint_id(); OwrCodecType _owr_caps_to_codec_type(GstCaps *caps); +const gchar* _owr_codec_type_to_caps_mime(OwrMediaType media_type, OwrCodecType codec_type); +gpointer _owr_detect_codecs(gpointer data); +const GList *_owr_get_detected_h264_encoders(); +const GList *_owr_get_detected_vp8_encoders(); +const GList *_owr_get_detected_vp9_encoders(); +GstElement *_owr_try_codecs(const GList *codecs, const gchar *name_prefix); +GstElement *_owr_create_decoder(OwrCodecType codec_type); +GstElement *_owr_create_parser(OwrCodecType codec_type); +const gchar* _owr_get_encoder_name(OwrCodecType codec_type); +void _owr_bin_link_and_sync_elements(GstBin *bin, gboolean *out_link_ok, gboolean *out_sync_ok, GstElement **out_first, GstElement **out_last); void _owr_utils_call_closure_with_list(GClosure *callback, GList *list); GClosure *_owr_utils_list_closure_merger_new(GClosure *final_callback, GCopyFunc list_item_copy, @@ -59,6 +72,7 @@ gboolean _owr_gst_caps_foreach(const GstCaps *caps, OwrGstCapsForeachFunc func, void _owr_deep_notify(GObject *object, GstObject *orig, GParamSpec *pspec, gpointer user_data); +void _owr_update_flip_method(GObject *renderer, GParamSpec *pspec, GstElement *flip); int _owr_rotation_and_mirror_to_video_flip_method(guint rotation, gboolean mirror); GHashTable *_owr_value_table_new(); diff --git a/tests/Makefile.am b/tests/Makefile.am index 28ac557f..cd2cc590 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -20,10 +20,9 @@ bin_PROGRAMS = \ test-send-receive \ test-data-channel \ test-init \ - test-bus \ test-uri \ - test-client \ - test-crypto-utils + test-crypto-utils \ + test-bus if OWR_GST AM_CPPFLAGS += \ @@ -45,6 +44,27 @@ test_gst_io_LDADD = \ $(top_builddir)/gst/libopenwebrtc_gst.la endif +if HAVE_JSON_GLIB +bin_PROGRAMS += \ + test-client + +test_client_SOURCES = test_client.c + +test_client_CFLAGS = \ + $(AM_CFLAGS) \ + $(JSON_GLIB_CFLAGS) \ + $(LIBSOUP_CFLAGS) \ + -I$(top_srcdir)/local \ + -I$(top_srcdir)/transport \ + -I$(top_srcdir)/owr + +test_client_LDADD = \ + $(JSON_GLIB_LIBS) \ + $(LIBSOUP_LIBS) \ + $(GLIB_LIBS) \ + $(top_builddir)/owr/libopenwebrtc.la +endif + list_devices_SOURCES = list_devices.c list_devices_CFLAGS = \ @@ -113,22 +133,6 @@ test_bus_LDADD = \ $(GLIB_LIBS) \ $(top_builddir)/owr/libopenwebrtc.la -test_client_SOURCES = test_client.c - -test_client_CFLAGS = \ - $(AM_CFLAGS) \ - $(JSON_GLIB_CFLAGS) \ - $(LIBSOUP_CFLAGS) \ - -I$(top_srcdir)/local \ - -I$(top_srcdir)/transport \ - -I$(top_srcdir)/owr - -test_client_LDADD = \ - $(JSON_GLIB_LIBS) \ - $(LIBSOUP_LIBS) \ - $(GLIB_LIBS) \ - $(top_builddir)/owr/libopenwebrtc.la - test_uri_SOURCES = test_uri.c test_utils.c test_uri_CFLAGS = \ diff --git a/tests/test_client.c b/tests/test_client.c index 520f647e..19f6fe2e 100644 --- a/tests/test_client.c +++ b/tests/test_client.c @@ -610,6 +610,7 @@ static void handle_offer(JsonReader *reader) g_object_set_data(G_OBJECT(transport_agent), "media-sessions", media_sessions); owr_transport_agent_add_session(transport_agent, OWR_SESSION(media_session)); } + owr_transport_agent_start(transport_agent); json_reader_end_member(reader); } @@ -771,7 +772,7 @@ static void send_eventsource_request(const gchar *url) static void got_local_sources(GList *sources, gchar *url) { local_sources = g_list_copy(sources); - transport_agent = owr_transport_agent_new(FALSE); + transport_agent = owr_transport_agent_new(FALSE, OWR_BUNDLE_POLICY_TYPE_BALANCED); owr_transport_agent_add_helper_server(transport_agent, OWR_HELPER_SERVER_TYPE_STUN, "stun.services.mozilla.com", 3478, NULL, NULL); if (url) { diff --git a/tests/test_client.py b/tests/test_client.py index 759ae8a2..664661fb 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -289,7 +289,7 @@ def handle_offer(message): session.set_send_source(source) ALL_SESSIONS.append((session, session_data)) TRANSPORT_AGENT.add_session(session) - + TRANSPORT_AGENT.start() def handle_remote_candidate(message): print("Handling remote candidate") diff --git a/tests/test_data_channel.c b/tests/test_data_channel.c index 2656dff3..3d179bf3 100644 --- a/tests/test_data_channel.c +++ b/tests/test_data_channel.c @@ -223,14 +223,14 @@ static gboolean setup_transport_agents() { g_print("Setting up transport agents\n"); // LEFT - left_transport_agent = owr_transport_agent_new(FALSE); + left_transport_agent = owr_transport_agent_new(FALSE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(left_transport_agent)); owr_transport_agent_set_local_port_range(left_transport_agent, 5000, 5999); owr_transport_agent_add_local_address(left_transport_agent, "127.0.0.1"); // RIGHT - right_transport_agent = owr_transport_agent_new(TRUE); + right_transport_agent = owr_transport_agent_new(TRUE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(right_transport_agent)); owr_transport_agent_set_local_port_range(right_transport_agent, 5000, 5999); @@ -247,7 +247,9 @@ static gboolean setup_transport_agents() owr_transport_agent_add_session(left_transport_agent, OWR_SESSION(left_session)); owr_transport_agent_add_session(right_transport_agent, OWR_SESSION(right_session)); - + owr_transport_agent_start(left_transport_agent); + owr_transport_agent_start(right_transport_agent); + if (wait_for_dtls) { gboolean peer_certificate_received; GAsyncQueue *msg_queue = g_async_queue_new(); diff --git a/tests/test_self_view.c b/tests/test_self_view.c index b83b7747..8d53f4b7 100644 --- a/tests/test_self_view.c +++ b/tests/test_self_view.c @@ -49,7 +49,12 @@ static OwrMediaRenderer *audio_renderer = NULL, *video_renderer = NULL; gboolean dump_pipeline(gpointer user_data) { - g_print("Dumping pipelines\n"); + if (!g_getenv("OWR_DEBUG_DUMP_DOT_DIR")) { + g_print("Not dumping pipelines because OWR_DEBUG_DUMP_DOT_DIR environment variable is empty.\n"); + return FALSE; + } + + g_print("Dumping pipelines..."); if (audio_source) write_dot_file("test_self_view-audio_source", owr_media_source_get_dot_data(audio_source), FALSE); @@ -61,6 +66,8 @@ gboolean dump_pipeline(gpointer user_data) if (video_renderer) write_dot_file("test_self_view-video_renderer", owr_media_renderer_get_dot_data(video_renderer), FALSE); + g_print(" done.\n"); + return FALSE; } diff --git a/tests/test_send_receive.c b/tests/test_send_receive.c index 35fb0c77..f4bb44ea 100644 --- a/tests/test_send_receive.c +++ b/tests/test_send_receive.c @@ -236,7 +236,7 @@ static void got_sources(GList *sources, gpointer user_data) owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(source)); - payload = owr_video_payload_new(OWR_CODEC_TYPE_VP8, 103, 90000, TRUE, FALSE); + payload = owr_video_payload_new(OWR_CODEC_TYPE_H264, 103, 90000, TRUE, FALSE); g_object_set(payload, "width", 640, "height", 480, "framerate", 30.0, NULL); g_object_set(payload, "rtx-payload-type", 123, NULL); if (adaptation) @@ -281,6 +281,8 @@ static void got_sources(GList *sources, gpointer user_data) sources = sources->next; } + owr_transport_agent_start(recv_transport_agent); + owr_transport_agent_start(send_transport_agent); } void on_new_source(gpointer *unused, OwrMediaSource *source) @@ -412,7 +414,7 @@ int main(int argc, char **argv) owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(owr_window_registry_get())); - recv_transport_agent = owr_transport_agent_new(FALSE); + recv_transport_agent = owr_transport_agent_new(FALSE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(recv_transport_agent)); owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(recv_transport_agent)); @@ -423,7 +425,7 @@ int main(int argc, char **argv) owr_transport_agent_add_local_address(recv_transport_agent, local_addr); // SEND - send_transport_agent = owr_transport_agent_new(TRUE); + send_transport_agent = owr_transport_agent_new(TRUE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(send_transport_agent)); owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(send_transport_agent)); diff --git a/transport/owr_crypto_utils.c b/transport/owr_crypto_utils.c index 785fde84..60872a47 100644 --- a/transport/owr_crypto_utils.c +++ b/transport/owr_crypto_utils.c @@ -148,8 +148,22 @@ gpointer _create_crypto_worker_run(gpointer data) key_pair = EVP_PKEY_new(); + // RSA_generate_key was deprecated in OpenSSL 0.9.8. +#if OPENSSL_VERSION_NUMBER < 0x10100001L rsa = RSA_generate_key(2048, RSA_F4, NULL, NULL); - +#else + rsa = RSA_new (); + if (rsa != NULL) { + BIGNUM *e = BN_new (); + if (e == NULL || !BN_set_word(e, RSA_F4) + || !RSA_generate_key_ex(rsa, 2048, e, NULL)) { + RSA_free(rsa); + rsa = NULL; + } + if (e) + BN_free(e); + } +#endif EVP_PKEY_assign_RSA(key_pair, rsa); X509_set_version(cert, 2); diff --git a/transport/owr_media_session.c b/transport/owr_media_session.c index adb2bc2a..e024169b 100644 --- a/transport/owr_media_session.c +++ b/transport/owr_media_session.c @@ -66,6 +66,8 @@ struct _OwrMediaSessionPrivate { gchar *incoming_srtp_key; gchar *outgoing_srtp_key; guint send_ssrc; + guint receive_ssrc; + guint receive_rtx_ssrc; gchar *cname; GRWLock rw_lock; OwrPayload *send_payload; @@ -94,6 +96,8 @@ enum { PROP_INCOMING_SRTP_KEY, PROP_OUTGOING_SRTP_KEY, PROP_SEND_SSRC, + PROP_RECEIVE_SSRC, + PROP_RECEIVE_RTX_SSRC, PROP_CNAME, PROP_JITTER_BUFFER_LATENCY, @@ -138,6 +142,14 @@ static void owr_media_session_set_property(GObject *object, guint property_id, c priv->send_ssrc = g_value_get_uint(value); break; + case PROP_RECEIVE_SSRC: + priv->receive_ssrc = g_value_get_uint(value); + break; + + case PROP_RECEIVE_RTX_SSRC: + priv->receive_rtx_ssrc = g_value_get_uint(value); + break; + case PROP_CNAME: priv->cname = g_value_dup_string(value); break; @@ -173,6 +185,14 @@ static void owr_media_session_get_property(GObject *object, guint property_id, G g_value_set_uint(value, priv->send_ssrc); break; + case PROP_RECEIVE_SSRC: + g_value_set_uint(value, priv->receive_ssrc); + break; + + case PROP_RECEIVE_RTX_SSRC: + g_value_set_uint(value, priv->receive_rtx_ssrc); + break; + case PROP_CNAME: g_value_set_string(value, priv->cname); break; @@ -294,6 +314,14 @@ static void owr_media_session_class_init(OwrMediaSessionClass *klass) "The ssrc (to be) used for the outgoing RTP media stream", 0, G_MAXUINT, 0, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_RECEIVE_SSRC] = g_param_spec_uint("receive-ssrc", "Receive ssrc", + "The ssrc (to be) used for the incoming RTP media stream", + 0, G_MAXUINT, 0, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + + obj_properties[PROP_RECEIVE_RTX_SSRC] = g_param_spec_uint("receive-rtx-ssrc", "Receive RTX ssrc", + "The ssrc (to be) used for the incoming RTP RTX media stream", + 0, G_MAXUINT, 0, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_CNAME] = g_param_spec_string("cname", "CNAME", "The canonical name identifying this endpoint", NULL, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); @@ -317,6 +345,8 @@ static void owr_media_session_init(OwrMediaSession *media_session) priv->incoming_srtp_key = NULL; priv->outgoing_srtp_key = NULL; priv->send_ssrc = 0; + priv->receive_ssrc = 0; + priv->receive_rtx_ssrc = 0; priv->cname = NULL; priv->send_payload = NULL; priv->send_source = NULL; diff --git a/transport/owr_payload.c b/transport/owr_payload.c index 151c1721..a43ce3bb 100644 --- a/transport/owr_payload.c +++ b/transport/owr_payload.c @@ -92,8 +92,6 @@ enum { static GParamSpec *obj_properties[N_PROPERTIES] = {NULL, }; -static guint get_unique_id(); - static void owr_payload_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { @@ -183,49 +181,6 @@ static void owr_payload_get_property(GObject *object, guint property_id, GValue } } -/* To be extended once more codecs are supported */ -static GList *h264_decoders = NULL; -static GList *h264_encoders = NULL; -static GList *vp8_decoders = NULL; -static GList *vp8_encoders = NULL; - -static gpointer owr_payload_detect_codecs(gpointer data) -{ - GList *decoder_factories; - GList *encoder_factories; - GstCaps *caps; - - OWR_UNUSED(data); - - decoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | - GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, - GST_RANK_MARGINAL); - encoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER | - GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, - GST_RANK_MARGINAL); - - caps = gst_caps_new_empty_simple("video/x-h264"); - h264_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); - h264_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); - gst_caps_unref(caps); - - caps = gst_caps_new_empty_simple("video/x-vp8"); - vp8_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); - vp8_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); - gst_caps_unref(caps); - - gst_plugin_feature_list_free(decoder_factories); - gst_plugin_feature_list_free(encoder_factories); - - h264_decoders = g_list_sort(h264_decoders, gst_plugin_feature_rank_compare_func); - h264_encoders = g_list_sort(h264_encoders, gst_plugin_feature_rank_compare_func); - vp8_decoders = g_list_sort(vp8_decoders, gst_plugin_feature_rank_compare_func); - vp8_encoders = g_list_sort(vp8_encoders, gst_plugin_feature_rank_compare_func); - - return NULL; -} - - static void owr_payload_class_init(OwrPayloadClass *klass) { GObjectClass *gobject_class = G_OBJECT_CLASS(klass); @@ -284,10 +239,6 @@ static void owr_payload_class_init(OwrPayloadClass *klass) static void owr_payload_init(OwrPayload *payload) { - static GOnce g_once = G_ONCE_INIT; - - g_once(&g_once, owr_payload_detect_codecs, NULL); - payload->priv = OWR_PAYLOAD_GET_PRIVATE(payload); payload->priv->mtu = DEFAULT_MTU; payload->priv->bitrate = DEFAULT_BITRATE; @@ -300,11 +251,8 @@ static void owr_payload_init(OwrPayload *payload) /* Private methods */ -static const gchar *OwrCodecTypeEncoderElementName[] = {"none", "mulawenc", "alawenc", "opusenc", "openh264enc", "vp8enc"}; -static const gchar *OwrCodecTypeDecoderElementName[] = {"none", "mulawdec", "alawdec", "opusdec", "openh264dec", "vp8dec"}; -static const gchar *OwrCodecTypeParserElementName[] = {"none", "none", "none", "none", "h264parse", "none"}; -static const gchar *OwrCodecTypePayElementName[] = {"none", "rtppcmupay", "rtppcmapay", "rtpopuspay", "rtph264pay", "rtpvp8pay"}; -static const gchar *OwrCodecTypeDepayElementName[] = {"none", "rtppcmudepay", "rtppcmadepay", "rtpopusdepay", "rtph264depay", "rtpvp8depay"}; +static const gchar *OwrCodecTypePayElementName[] = { NULL, "rtppcmupay", "rtppcmapay", "rtpopuspay", "rtph264pay", "rtpvp8pay", "rtpvp9pay" }; +static const gchar *OwrCodecTypeDepayElementName[] = { NULL, "rtppcmudepay", "rtppcmadepay", "rtpopusdepay", "rtph264depay", "rtpvp8depay", "rtpvp9depay" }; static guint evaluate_bitrate_from_payload(OwrPayload *payload) { @@ -327,40 +275,6 @@ static guint evaluate_bitrate_from_payload(OwrPayload *payload) return bitrate; } -static GstElement * try_codecs(GList *codecs, const gchar *name_prefix) -{ - GList *l; - gchar *element_name; - - for (l = codecs; l; l = l->next) { - GstElementFactory *f = l->data; - GstElement *e; - - element_name = g_strdup_printf("%s_%s_%u", name_prefix, - gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(f)), - get_unique_id()); - - e = gst_element_factory_create(f, element_name); - g_free(element_name); - - if (!e) - continue; - - /* Try setting to READY. If this fails the codec does not work, for - * example because the hardware codec is currently busy - */ - if (gst_element_set_state(e, GST_STATE_READY) != GST_STATE_CHANGE_SUCCESS) { - gst_element_set_state(e, GST_STATE_NULL); - gst_object_unref(e); - continue; - } - - return e; - } - - return NULL; -} - static gboolean binding_transform_to_kbps(GBinding *binding, const GValue *from_value, GValue *to_value, gpointer user_data) { guint bitrate; @@ -386,7 +300,7 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) switch (payload->priv->codec_type) { case OWR_CODEC_TYPE_H264: - encoder = try_codecs(h264_encoders, "encoder"); + encoder = _owr_try_codecs(_owr_get_detected_h264_encoders(), "encoder"); g_return_val_if_fail(encoder, NULL); factory = gst_element_get_factory(encoder); @@ -415,7 +329,7 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) #endif "max-keyframe-interval", G_MAXINT, NULL); - } else { + } else if (strcmp(factory_name, "omxh264enc")) { /* Assume bits/s instead of kbit/s */ g_object_bind_property(payload, "bitrate", encoder, "bitrate", G_BINDING_SYNC_CREATE); } @@ -423,7 +337,7 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) break; case OWR_CODEC_TYPE_VP8: - encoder = try_codecs(vp8_encoders, "encoder"); + encoder = _owr_try_codecs(_owr_get_detected_vp8_encoders(), "encoder"); g_return_val_if_fail(encoder, NULL); #if (defined(__APPLE__) && TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR) || defined(__ANDROID__) @@ -450,66 +364,39 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) g_object_bind_property(payload, "bitrate", encoder, "target-bitrate", G_BINDING_SYNC_CREATE); g_object_set(payload, "bitrate", evaluate_bitrate_from_payload(payload), NULL); break; - default: - element_name = g_strdup_printf("encoder_%s_%u", OwrCodecTypeEncoderElementName[payload->priv->codec_type], get_unique_id()); - encoder = gst_element_factory_make(OwrCodecTypeEncoderElementName[payload->priv->codec_type], element_name); - g_free(element_name); + case OWR_CODEC_TYPE_VP9: + encoder = _owr_try_codecs(_owr_get_detected_vp9_encoders(), "encoder"); g_return_val_if_fail(encoder, NULL); - break; - } - - return encoder; -} - -GstElement * _owr_payload_create_decoder(OwrPayload *payload) -{ - GstElement * decoder = NULL; - gchar *element_name = NULL; - - g_return_val_if_fail(payload, NULL); + /* values are inspired by webrtc.org values in vp9_impl.cc */ + g_object_set(encoder, + "end-usage", 1, /* VPX_CBR */ + "deadline", G_GINT64_CONSTANT(1), /* VPX_DL_REALTIME */ + "cpu-used", 3, + "min-quantizer", 2, + "max-quantizer", 52, + "buffer-initial-size", 500, + "buffer-optimal-size", 600, + "buffer-size", 1000, + "dropframe-threshold", 30, + "lag-in-frames", 0, + "timebase", 1, 90000, + "error-resilient", 1, + "resize-allowed", TRUE, + "keyframe-mode", 0, /* VPX_KF_DISABLED */ + NULL); - switch (payload->priv->codec_type) { - case OWR_CODEC_TYPE_H264: - decoder = try_codecs(h264_decoders, "decoder"); - g_return_val_if_fail(decoder, NULL); - break; - case OWR_CODEC_TYPE_VP8: - decoder = try_codecs(vp8_decoders, "decoder"); - g_return_val_if_fail(decoder, NULL); + g_object_bind_property(payload, "bitrate", encoder, "target-bitrate", G_BINDING_SYNC_CREATE); + g_object_set(payload, "bitrate", evaluate_bitrate_from_payload(payload), NULL); break; default: - element_name = g_strdup_printf("decoder_%s_%u", OwrCodecTypeDecoderElementName[payload->priv->codec_type], get_unique_id()); - decoder = gst_element_factory_make(OwrCodecTypeDecoderElementName[payload->priv->codec_type], element_name); + element_name = g_strdup_printf("encoder_%s_%u", _owr_get_encoder_name(payload->priv->codec_type), _owr_get_unique_uint_id()); + encoder = gst_element_factory_make(_owr_get_encoder_name(payload->priv->codec_type), element_name); g_free(element_name); - g_return_val_if_fail(decoder, NULL); + g_return_val_if_fail(encoder, NULL); break; } - return decoder; -} - -GstElement * _owr_payload_create_parser(OwrPayload *payload) -{ - GstElement * parser = NULL; - gchar *element_name = NULL; - - g_return_val_if_fail(payload, NULL); - - if (!g_strcmp0(OwrCodecTypeParserElementName[payload->priv->codec_type], "none")) - return NULL; - - element_name = g_strdup_printf("parser_%s_%u", OwrCodecTypeParserElementName[payload->priv->codec_type], get_unique_id()); - parser = gst_element_factory_make(OwrCodecTypeParserElementName[payload->priv->codec_type], element_name); - g_free(element_name); - - switch (payload->priv->codec_type) { - case OWR_CODEC_TYPE_H264: - g_object_set(parser, "disable-passthrough", TRUE, NULL); - break; - default: - break; - } - return parser; + return encoder; } GstElement * _owr_payload_create_payload_packetizer(OwrPayload *payload) @@ -520,7 +407,7 @@ GstElement * _owr_payload_create_payload_packetizer(OwrPayload *payload) g_return_val_if_fail(payload, NULL); - element_name = g_strdup_printf("pay_%s_%u", OwrCodecTypePayElementName[payload->priv->codec_type], get_unique_id()); + element_name = g_strdup_printf("pay_%s_%u", OwrCodecTypePayElementName[payload->priv->codec_type], _owr_get_unique_uint_id()); pay = gst_element_factory_make(OwrCodecTypePayElementName[payload->priv->codec_type], element_name); g_free(element_name); @@ -556,7 +443,7 @@ GstElement * _owr_payload_create_payload_depacketizer(OwrPayload *payload) g_return_val_if_fail(payload, NULL); - element_name = g_strdup_printf("depay_%s_%u", OwrCodecTypeDepayElementName[payload->priv->codec_type], get_unique_id()); + element_name = g_strdup_printf("depay_%s_%u", OwrCodecTypeDepayElementName[payload->priv->codec_type], _owr_get_unique_uint_id()); depay = gst_element_factory_make(OwrCodecTypeDepayElementName[payload->priv->codec_type], element_name); g_free(element_name); @@ -572,6 +459,14 @@ OwrMediaType _owr_payload_get_media_type(OwrPayload *payload) return media_type; } +OwrCodecType _owr_payload_get_codec_type(OwrPayload *payload) +{ + OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; + + g_object_get(payload, "codec-type", &codec_type, NULL); + return codec_type; +} + GstCaps * _owr_payload_create_rtp_caps(OwrPayload *payload) { @@ -607,6 +502,10 @@ GstCaps * _owr_payload_create_rtp_caps(OwrPayload *payload) encoding_name = "VP8-DRAFT-IETF-01"; break; + case OWR_CODEC_TYPE_VP9: + encoding_name = "VP9-DRAFT-IETF-01"; + break; + default: g_return_val_if_reached(NULL); } @@ -718,6 +617,9 @@ GstCaps * _owr_payload_create_encoded_caps(OwrPayload *payload) case OWR_CODEC_TYPE_VP8: caps = gst_caps_new_empty_simple("video/x-vp8"); break; + case OWR_CODEC_TYPE_VP9: + caps = gst_caps_new_empty_simple("video/x-vp9"); + break; default: caps = gst_caps_new_any(); } @@ -725,12 +627,40 @@ GstCaps * _owr_payload_create_encoded_caps(OwrPayload *payload) return caps; } +gboolean owr_payload_supported(OwrCodecType codec_type) +{ + gboolean supported = FALSE; + GstElement* encoder = NULL; + GstElement* decoder = _owr_create_decoder(codec_type); + + switch (codec_type) { + case OWR_CODEC_TYPE_H264: + encoder = _owr_try_codecs(_owr_get_detected_h264_encoders(), NULL); + break; + case OWR_CODEC_TYPE_VP8: + encoder = _owr_try_codecs(_owr_get_detected_vp8_encoders(), NULL); + break; + case OWR_CODEC_TYPE_VP9: + encoder = _owr_try_codecs(_owr_get_detected_vp9_encoders(), NULL); + break; + default: + encoder = gst_element_factory_make(_owr_get_encoder_name(codec_type), NULL); + } + + supported = encoder && decoder; + + if (encoder) { + gst_element_set_state(encoder, GST_STATE_NULL); + gst_object_unref(encoder); + } + + if (decoder) { + gst_element_set_state(decoder, GST_STATE_NULL); + gst_object_unref(decoder); + } + + return supported; +} /* local functions */ - -static guint get_unique_id() -{ - static guint id = 0; - return g_atomic_int_add(&id, 1); -} diff --git a/transport/owr_payload.h b/transport/owr_payload.h index 6de7d524..66d64090 100644 --- a/transport/owr_payload.h +++ b/transport/owr_payload.h @@ -65,6 +65,8 @@ struct _OwrPayloadClass { GType owr_payload_get_type(void) G_GNUC_CONST; +gboolean owr_payload_supported(OwrCodecType codec_type); + G_END_DECLS #endif /* __OWR_PAYLOAD_H__ */ diff --git a/transport/owr_payload_private.h b/transport/owr_payload_private.h index 8f9ec579..b1deb25e 100644 --- a/transport/owr_payload_private.h +++ b/transport/owr_payload_private.h @@ -38,11 +38,10 @@ G_BEGIN_DECLS /*< private >*/ GstElement * _owr_payload_create_encoder(OwrPayload *payload); -GstElement * _owr_payload_create_decoder(OwrPayload *payload); -GstElement * _owr_payload_create_parser(OwrPayload *payload); GstElement * _owr_payload_create_payload_packetizer(OwrPayload *payload); GstElement * _owr_payload_create_payload_depacketizer(OwrPayload *payload); OwrMediaType _owr_payload_get_media_type(OwrPayload *payload); +OwrCodecType _owr_payload_get_codec_type(OwrPayload *payload); GstCaps * _owr_payload_create_rtp_caps(OwrPayload *payload); GstCaps * _owr_payload_create_raw_caps(OwrPayload *payload); GstCaps * _owr_payload_create_encoded_caps(OwrPayload *payload); diff --git a/transport/owr_remote_media_source.c b/transport/owr_remote_media_source.c index d2f62f15..e913a1c1 100644 --- a/transport/owr_remote_media_source.c +++ b/transport/owr_remote_media_source.c @@ -79,7 +79,9 @@ static void on_caps(GstElement *source, GParamSpec *pspec, OwrMediaSource *media if (GST_IS_CAPS(caps)) { GST_INFO_OBJECT(source, "%s - configured with caps: %" GST_PTR_FORMAT, media_source_name, caps); + gst_caps_unref(caps); } + g_free(media_source_name); } #define LINK_ELEMENTS(a, b) \ @@ -87,7 +89,7 @@ static void on_caps(GstElement *source, GParamSpec *pspec, OwrMediaSource *media GST_ERROR("Failed to link " #a " -> " #b); OwrMediaSource *_owr_remote_media_source_new(OwrMediaType media_type, - guint stream_id, OwrCodecType codec_type, gpointer transport_bin_ptr) + guint session_id, guint stream_id, OwrCodecType codec_type, gpointer transport_bin_ptr) { GstElement *transport_bin = GST_ELEMENT(transport_bin_ptr); OwrRemoteMediaSource *source; @@ -119,10 +121,10 @@ OwrMediaSource *_owr_remote_media_source_new(OwrMediaType media_type, /* create source tee and everything */ if (media_type == OWR_MEDIA_TYPE_VIDEO) { bin_name = g_strdup_printf("video-src-%u-%u", codec_type, stream_id); - pad_name = g_strdup_printf("video_src_%u_%u", codec_type, stream_id); + pad_name = g_strdup_printf("video_src_%u_%u_%u", codec_type, session_id, stream_id); } else if (media_type == OWR_MEDIA_TYPE_AUDIO) { bin_name = g_strdup_printf("audio-src-%u-%u", codec_type, stream_id); - pad_name = g_strdup_printf("audio_raw_src_%u", stream_id); + pad_name = g_strdup_printf("audio_raw_src_%u_%u", session_id, stream_id); } else g_assert_not_reached(); diff --git a/transport/owr_remote_media_source_private.h b/transport/owr_remote_media_source_private.h index 1729d3b4..3cb7b84d 100644 --- a/transport/owr_remote_media_source_private.h +++ b/transport/owr_remote_media_source_private.h @@ -40,7 +40,7 @@ G_BEGIN_DECLS OwrMediaSource *_owr_remote_media_source_new(OwrMediaType media_type, - guint stream_id, OwrCodecType codec_type, gpointer transport_bin); + guint session_id, guint stream_id, OwrCodecType codec_type, gpointer transport_bin); G_END_DECLS diff --git a/transport/owr_session.c b/transport/owr_session.c index 6d7a3a13..d0109ac8 100644 --- a/transport/owr_session.c +++ b/transport/owr_session.c @@ -66,6 +66,7 @@ G_DEFINE_TYPE_WITH_CODE(OwrSession, owr_session, G_TYPE_OBJECT, G_IMPLEMENT_INTERFACE(OWR_TYPE_MESSAGE_ORIGIN, owr_message_origin_interface_init)) struct _OwrSessionPrivate { + guint stream_id; gboolean dtls_client_mode; gchar *dtls_certificate; gchar *dtls_key; @@ -332,6 +333,7 @@ static void owr_session_init(OwrSession *session) OwrSessionPrivate *priv; session->priv = priv = OWR_SESSION_GET_PRIVATE(session); + priv->stream_id = 0; priv->dtls_client_mode = DEFAULT_DTLS_CLIENT_MODE; priv->dtls_certificate = g_strdup(DEFAULT_DTLS_CERTIFICATE); priv->dtls_key = g_strdup(DEFAULT_DTLS_KEY); @@ -776,3 +778,13 @@ void _owr_session_emit_ice_state_changed(OwrSession *session, guint session_id, session->priv->ice_state = new_state; g_object_notify_by_pspec(G_OBJECT(session), obj_properties[PROP_ICE_STATE]); } + +void _owr_session_set_stream_id(OwrSession *session, guint stream_id) +{ + session->priv->stream_id = stream_id; +} + +guint _owr_session_get_stream_id(OwrSession *session) +{ + return session->priv->stream_id; +} diff --git a/transport/owr_session_private.h b/transport/owr_session_private.h index b6adaba8..bceaee20 100644 --- a/transport/owr_session_private.h +++ b/transport/owr_session_private.h @@ -50,6 +50,9 @@ void _owr_session_set_dtls_peer_certificate(OwrSession *, const gchar *certifica void _owr_session_emit_ice_state_changed(OwrSession *session, guint session_id, OwrComponentType component_type, OwrIceState state); +void _owr_session_set_stream_id(OwrSession *session, guint session_id); +guint _owr_session_get_stream_id(OwrSession *session); + G_END_DECLS #endif /* __GTK_DOC_IGNORE__ */ diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index cffa2242..74a94a18 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -79,11 +80,13 @@ GST_DEBUG_CATEGORY_EXTERN(_owrsession_debug); #define GST_CAT_DEFAULT _owrtransportagent_debug #define DEFAULT_ICE_CONTROLLING_MODE TRUE +#define DEFAULT_BUNDLE_POLICY OWR_BUNDLE_POLICY_TYPE_BALANCED #define GST_RTCP_RTPFB_TYPE_SCREAM 18 enum { PROP_0, PROP_ICE_CONTROLLING_MODE, + PROP_BUNDLE_POLICY, N_PROPERTIES }; @@ -100,6 +103,7 @@ G_DEFINE_TYPE_WITH_CODE(OwrTransportAgent, owr_transport_agent, G_TYPE_OBJECT, typedef struct { OwrDataChannelState state; GstElement *data_sink, *data_src; + guint stream_id; guint session_id; gboolean ordered; @@ -134,7 +138,9 @@ typedef struct { struct _OwrTransportAgentPrivate { NiceAgent *nice_agent; + guint next_session_id; gboolean ice_controlling_mode; + gboolean bundle_policy; GMutex sessions_lock; GHashTable *sessions; @@ -163,6 +169,8 @@ struct _OwrTransportAgentPrivate { GRWLock data_channels_rw_mutex; gboolean data_session_added, data_session_established; OwrMessageOriginBusSet *message_origin_bus_set; + + GSList* unstarted_sessions; }; typedef struct { @@ -173,6 +181,7 @@ typedef struct { typedef struct { OwrTransportAgent *transport_agent; guint session_id; + guint stream_id; gint rtx_pt; gboolean adapt; @@ -185,7 +194,8 @@ typedef struct { guint32 last_feedback_wallclock; } ScreamRx; -#define GEN_HASH_KEY(seq, ssrc) (seq ^ ssrc) +#define AGENT_SESSIONS_LOCK(agent) g_mutex_lock(&agent->priv->sessions_lock); +#define AGENT_SESSIONS_UNLOCK(agent) g_mutex_unlock(&agent->priv->sessions_lock); static void owr_transport_agent_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec); @@ -196,14 +206,16 @@ static void owr_transport_agent_get_property(GObject *object, guint property_id, static void add_helper_server_info(GResolver *resolver, GAsyncResult *result, GHashTable *info); static void update_helper_servers(OwrTransportAgent *transport_agent, guint stream_id); static gboolean add_session(GHashTable *args); -static guint get_stream_id(OwrTransportAgent *transport_agent, OwrSession *session); -static OwrSession * get_session(OwrTransportAgent *transport_agent, guint stream_id); -static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_agent, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); -static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_agent, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); -static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *transport_agent, - guint stream_id); -static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transport_agent, - guint stream_id); +static guint get_session_id_unlocked(OwrTransportAgent *transport_agent, OwrSession *session); +static guint get_session_id(OwrTransportAgent *transport_agent, OwrSession *session); +static OwrSession * get_session_unlocked(OwrTransportAgent *transport_agent, guint session_id); +static OwrSession * get_session(OwrTransportAgent *transport_agent, guint session_id); +static OwrSession * get_session_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id); +static GSList * get_sessions_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id); +static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); +static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); +static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id); +static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id); static void set_send_ssrc_and_cname(OwrTransportAgent *agent, OwrMediaSession *media_session); static void on_new_candidate(NiceAgent *nice_agent, NiceCandidate *nice_candidate, OwrTransportAgent *transport_agent); static void on_candidate_gathering_done(NiceAgent *nice_agent, guint stream_id, OwrTransportAgent *transport_agent); @@ -214,11 +226,11 @@ static void on_local_candidate_change(OwrTransportAgent *transport_agent, OwrCan static void on_transport_bin_pad_added(GstElement *transport_bin, GstPad *new_pad, OwrTransportAgent *transport_agent); static void on_rtpbin_pad_added(GstElement *rtpbin, GstPad *new_pad, OwrTransportAgent *agent); -static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent); -static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent); -static GstCaps * on_rtpbin_request_pt_map(GstElement *rtpbin, guint session_id, guint pt, OwrTransportAgent *agent); -static GstElement * on_rtpbin_request_aux_sender(GstElement *rtpbin, guint session_id, OwrTransportAgent *transport_agent); -static GstElement * on_rtpbin_request_aux_receiver(GstElement *rtpbin, guint session_id, OwrTransportAgent *transport_agent); +static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent); +static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent); +static GstCaps * on_rtpbin_request_pt_map(GstElement *rtpbin, guint stream_id, guint pt, OwrTransportAgent *agent); +static GstElement * on_rtpbin_request_aux_sender(GstElement *rtpbin, guint stream_id, OwrTransportAgent *transport_agent); +static GstElement * on_rtpbin_request_aux_receiver(GstElement *rtpbin, guint stream_id, OwrTransportAgent *transport_agent); static void on_dtls_enc_key_set(GstElement *dtls_srtp_enc, AgentAndSessionIdPair *data); static void on_new_selected_pair(NiceAgent *nice_agent, guint stream_id, guint component_id, @@ -230,8 +242,9 @@ static void on_receiving_rtcp(GObject *session, GstBuffer *buffer, OwrTransportA static void on_feedback_rtcp(GObject *session, guint type, guint fbtype, guint sender_ssrc, guint media_ssrc, GstBuffer *fci, OwrTransportAgent *transport_agent); static GstPadProbeReturn probe_save_ts(GstPad *srcpad, GstPadProbeInfo *info, void *user_data); static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, ScreamRx *scream_rx); -static void on_ssrc_active(GstElement *rtpbin, guint session_id, guint ssrc, OwrTransportAgent *transport_agent); -static void on_new_jitterbuffer(GstElement *rtpbin, GstElement *jitterbuffer, guint session_id, guint ssrc, OwrTransportAgent *transport_agent); +static void on_ssrc_active(GstElement *rtpbin, guint stream_id, guint ssrc, OwrTransportAgent *transport_agent); +static guint on_bundled_ssrc(GstElement *rtpbin, guint ssrc, OwrTransportAgent *transport_agent); +static void on_new_jitterbuffer(GstElement *rtpbin, GstElement *jitterbuffer, guint stream_id, guint ssrc, OwrTransportAgent *transport_agent); static void prepare_rtcp_stats(OwrMediaSession *media_session, GObject *rtp_source); static void data_channel_free(DataChannel *data_channel); @@ -246,7 +259,7 @@ static void on_sctp_association_established(GstElement *sctpenc, gboolean establ static GstFlowReturn new_data_callback(GstAppSink *appsink, OwrTransportAgent *transport_agent); static gboolean create_datachannel_appsrc(OwrTransportAgent *transport_agent, OwrDataChannel *data_channel); -static gboolean is_valid_sctp_stream_id(OwrTransportAgent *transport_agent, guint32 session_id, +static gboolean is_valid_sctp_session_id(OwrTransportAgent *transport_agent, guint32 session_id, guint16 sctp_stream_id, gboolean remotly_initiated); static guint8 * create_datachannel_open_request(OwrDataChannelChannelType channel_type, guint32 reliability_param, guint16 priority, const gchar *label, guint16 label_len, @@ -292,6 +305,11 @@ static void owr_transport_agent_finalize(GObject *object) gst_element_set_state(priv->pipeline, GST_STATE_NULL); gst_object_unref(priv->pipeline); + if (priv->unstarted_sessions) { + g_slist_free_full(priv->unstarted_sessions, g_object_unref); + priv->unstarted_sessions = NULL; + } + sessions_list = g_hash_table_get_values(priv->sessions); for (item = sessions_list; item; item = item->next) { session = item->data; @@ -342,6 +360,10 @@ static void owr_transport_agent_class_init(OwrTransportAgentClass *klass) "Ice controlling mode", "Whether the ice agent is in controlling mode", DEFAULT_ICE_CONTROLLING_MODE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_BUNDLE_POLICY] = g_param_spec_enum("bundle-policy", "bundle-policy" + "Bundle policy of the data streams", "What kind of bundle policy we will use for the streams", + OWR_TYPE_BUNDLE_POLICY_TYPE, DEFAULT_BUNDLE_POLICY, G_PARAM_CONSTRUCT_ONLY | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + gobject_class->set_property = owr_transport_agent_set_property; gobject_class->get_property = owr_transport_agent_get_property; gobject_class->finalize = owr_transport_agent_finalize; @@ -434,8 +456,10 @@ static void owr_transport_agent_init(OwrTransportAgent *transport_agent) transport_agent->priv = priv = OWR_TRANSPORT_AGENT_GET_PRIVATE(transport_agent); priv->ice_controlling_mode = DEFAULT_ICE_CONTROLLING_MODE; + priv->bundle_policy = DEFAULT_BUNDLE_POLICY; priv->agent_id = next_transport_agent_id++; priv->nice_agent = NULL; + priv->next_session_id = 1; priv->sessions = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, g_object_unref); priv->pending_sessions = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, g_free); @@ -527,6 +551,11 @@ static void owr_transport_agent_set_property(GObject *object, guint property_id, case PROP_ICE_CONTROLLING_MODE: priv->ice_controlling_mode = g_value_get_boolean(value); break; + case PROP_BUNDLE_POLICY: + priv->bundle_policy = g_value_get_enum(value); + if (priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) + g_signal_connect(priv->rtpbin, "on-bundled-ssrc", G_CALLBACK(on_bundled_ssrc), transport_agent); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -548,16 +577,19 @@ static void owr_transport_agent_get_property(GObject *object, guint property_id, case PROP_ICE_CONTROLLING_MODE: g_value_set_boolean(value, priv->ice_controlling_mode); break; + case PROP_BUNDLE_POLICY: + g_value_set_enum(value, priv->bundle_policy); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; } } -OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode) +OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode, OwrBundlePolicyType bundle_policy_type) { return g_object_new(OWR_TYPE_TRANSPORT_AGENT, "ice-controlling_mode", ice_controlling_mode, - NULL); + "bundle-policy", bundle_policy_type, NULL); } /** @@ -631,19 +663,19 @@ void owr_transport_agent_set_local_port_range(OwrTransportAgent *transport_agent */ void owr_transport_agent_add_session(OwrTransportAgent *agent, OwrSession *session) { - GHashTable *args; - g_return_if_fail(agent); g_return_if_fail(OWR_IS_MEDIA_SESSION(session) || OWR_IS_DATA_SESSION(session)); - args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(agent)); - g_hash_table_insert(args, "transport_agent", agent); - g_hash_table_insert(args, "session", session); - - g_object_ref(agent); - - _owr_schedule_with_hash_table((GSourceFunc)add_session, args); + if (GST_STATE(agent->priv->pipeline) >= GST_STATE_READY) { + GHashTable *args; + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(agent)); + g_hash_table_insert(args, "transport_agent", agent); + g_hash_table_insert(args, "session", g_object_ref(session)); + g_object_ref(agent); + _owr_schedule_with_hash_table((GSourceFunc) add_session, args); + } else + agent->priv->unstarted_sessions = g_slist_append(agent->priv->unstarted_sessions, g_object_ref(session)); } @@ -653,12 +685,15 @@ static void add_helper_server_info(GResolver *resolver, GAsyncResult *result, GH { OwrTransportAgent *transport_agent; OwrTransportAgentPrivate *priv; - GList *stream_ids, *item, *address_list; + GList *address_list; GHashTable *stats_table; OwrMessageOrigin *message_origin; GValue *value; - guint stream_id; GError *error = NULL; + GHashTableIter iter; + gpointer key, session; + GList *stream_id_list = NULL; + GList *list_iter; transport_agent = OWR_TRANSPORT_AGENT(g_hash_table_lookup(info, "transport_agent")); g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); @@ -697,17 +732,22 @@ static void add_helper_server_info(GResolver *resolver, GAsyncResult *result, GH priv->helper_server_infos = g_list_append(priv->helper_server_infos, info); } - g_mutex_lock(&priv->sessions_lock); - stream_ids = g_hash_table_get_keys(priv->sessions); - g_mutex_unlock(&priv->sessions_lock); - for (item = stream_ids; item; item = item->next) { - stream_id = GPOINTER_TO_UINT(item->data); + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, priv->sessions); + while (g_hash_table_iter_next (&iter, &key, &session)) { + stream_id_list = g_list_prepend(stream_id_list, + GUINT_TO_POINTER(_owr_session_get_stream_id(OWR_SESSION(session)))); + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + for (list_iter = stream_id_list; list_iter != NULL; list_iter = list_iter->next) { if (info) - update_helper_servers(transport_agent, stream_id); + update_helper_servers(transport_agent, GPOINTER_TO_UINT(list_iter->data)); if (!priv->deferred_helper_server_adds) - nice_agent_gather_candidates(priv->nice_agent, stream_id); + nice_agent_gather_candidates(priv->nice_agent, GPOINTER_TO_UINT(list_iter->data)); } - g_list_free(stream_ids); + + g_list_free(stream_id_list); g_object_unref(transport_agent); @@ -761,6 +801,12 @@ static void update_helper_servers(OwrTransportAgent *transport_agent, guint stre nice_agent_set_relay_info(priv->nice_agent, stream_id, NICE_COMPONENT_TYPE_RTCP, address, port, username, password, NICE_RELAY_TYPE_TURN_TCP); break; + case OWR_HELPER_SERVER_TYPE_TURN_TLS: + nice_agent_set_relay_info(priv->nice_agent, stream_id, NICE_COMPONENT_TYPE_RTP, + address, port, username, password, NICE_RELAY_TYPE_TURN_TLS); + nice_agent_set_relay_info(priv->nice_agent, stream_id, NICE_COMPONENT_TYPE_RTCP, + address, port, username, password, NICE_RELAY_TYPE_TURN_TLS); + break; } } } @@ -778,7 +824,7 @@ static gboolean link_source_to_transport_bin(GstPad *srcpad, GstElement *pipelin if (media_type == OWR_MEDIA_TYPE_VIDEO) g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_sink_%u_%u", codec_type, stream_id); else if (media_type == OWR_MEDIA_TYPE_AUDIO) - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "audio_raw_sink_%u", stream_id); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "audio_sink_%u_%u", codec_type, stream_id); sinkpad = gst_element_get_static_pad(transport_bin, name); ret = gst_pad_link(srcpad, sinkpad) == GST_PAD_LINK_OK; @@ -816,10 +862,7 @@ static void handle_new_send_source(OwrTransportAgent *transport_agent, return; } - /* FIXME - communicate what codec types are supported by the source - * and if one is reusable, use it, else raw? g_object_get(send_payload, "codec-type", &codec_type, NULL); - */ caps = _owr_payload_create_raw_caps(send_payload); src = _owr_media_source_request_source(send_source, caps); @@ -829,8 +872,7 @@ static void handle_new_send_source(OwrTransportAgent *transport_agent, g_assert(srcpad); transport_bin = transport_agent->priv->transport_bin; - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); gst_bin_add(GST_BIN(transport_agent->priv->pipeline), src); if (!link_source_to_transport_bin(srcpad, transport_agent->priv->pipeline, transport_bin, media_type, codec_type, stream_id)) { @@ -843,25 +885,13 @@ static void handle_new_send_source(OwrTransportAgent *transport_agent, gst_element_sync_state_with_parent(src); } -static void maybe_handle_new_send_source_with_payload(OwrTransportAgent *transport_agent, - OwrMediaSession *media_session) +static void maybe_handle_new_send_source_with_payload_for_session(OwrTransportAgent *transport_agent, + OwrMediaSession *media_session, gboolean pending) { OwrPayload *payload = NULL; OwrMediaSource *media_source = NULL; GHashTable *event_data; GValue *value; - guint stream_id; - gboolean pending; - - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); - g_return_if_fail(stream_id); - - g_mutex_lock(&transport_agent->priv->sessions_lock); - pending = g_hash_table_lookup_extended(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id), NULL, NULL); - g_mutex_unlock(&transport_agent->priv->sessions_lock); - - g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); if (!pending && (payload = _owr_media_session_get_send_payload(media_session)) && @@ -885,16 +915,54 @@ static void maybe_handle_new_send_source_with_payload(OwrTransportAgent *transpo g_object_unref(media_source); } +static gboolean process_pending_bundled_session(gpointer key, gpointer value, gpointer user_data) +{ + guint session_id = GPOINTER_TO_UINT(key); + OwrSession* session; + OwrTransportAgent* transport_agent = (OwrTransportAgent*) user_data; + OWR_UNUSED(value); + + session = get_session_unlocked(transport_agent, session_id); + maybe_handle_new_send_source_with_payload_for_session(transport_agent, OWR_MEDIA_SESSION(session), FALSE); + g_object_unref(session); + return TRUE; +} + +static void maybe_handle_new_send_source_with_payload(OwrTransportAgent *transport_agent, + OwrMediaSession *media_session, guint process_bundled_sessions) +{ + g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); + g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); + + if (process_bundled_sessions) { + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_foreach_remove(transport_agent->priv->pending_sessions, (GHRFunc) process_pending_bundled_session, transport_agent); + AGENT_SESSIONS_UNLOCK(transport_agent); + } else { + gboolean pending; + guint stream_id; + + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); + AGENT_SESSIONS_LOCK(transport_agent); + pending = g_hash_table_lookup_extended(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id), NULL, NULL); + AGENT_SESSIONS_UNLOCK(transport_agent); + + maybe_handle_new_send_source_with_payload_for_session(transport_agent, media_session, pending); + } +} + static void remove_existing_send_source_and_payload(OwrTransportAgent *transport_agent, OwrMediaSource *media_source, OwrMediaSession *media_session) { - guint stream_id; + guint session_id, stream_id; gchar *pad_name = NULL, *bin_name; GstPad *bin_src_pad, *sinkpad; GstElement *send_input_bin, *source_bin; OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; GHashTable *event_data; GValue *value; + OwrPayload *send_payload; + OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; g_assert(media_source); @@ -902,17 +970,24 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport value = _owr_value_table_add(event_data, "start_time", G_TYPE_INT64); g_value_set_int64(value, g_get_monotonic_time()); + send_payload = _owr_media_session_get_send_payload(media_session); + if (send_payload) { + g_object_get(send_payload, "codec-type", &codec_type, NULL); + g_object_unref(send_payload); + } + /* Setting a new, different source but have one already */ - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); + session_id = get_session_id(transport_agent, OWR_SESSION(media_session)); /* Unlink the source bin */ g_object_get(media_source, "media-type", &media_type, NULL); g_warn_if_fail(media_type != OWR_MEDIA_TYPE_UNKNOWN); if (media_type == OWR_MEDIA_TYPE_VIDEO) - pad_name = g_strdup_printf("video_sink_%u_%u", OWR_CODEC_TYPE_NONE, stream_id); + pad_name = g_strdup_printf("video_sink_%u_%u", codec_type, stream_id); else - pad_name = g_strdup_printf("audio_raw_sink_%u", stream_id); + pad_name = g_strdup_printf("audio_sink_%u_%u", codec_type, stream_id); sinkpad = gst_element_get_static_pad(transport_agent->priv->transport_bin, pad_name); g_assert(sinkpad); g_free(pad_name); @@ -930,7 +1005,7 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport gst_object_unref(source_bin); /* Now the payload bin */ - bin_name = g_strdup_printf("send-input-bin-%u", stream_id); + bin_name = g_strdup_printf("send-input-bin-%u-%u", session_id, stream_id); send_input_bin = gst_bin_get_by_name(GST_BIN(transport_agent->priv->transport_bin), bin_name); g_assert(send_input_bin); g_free(bin_name); @@ -964,7 +1039,7 @@ static void on_new_send_payload(OwrTransportAgent *transport_agent, } if (new_payload && old_payload != new_payload) - maybe_handle_new_send_source_with_payload(transport_agent, media_session); + maybe_handle_new_send_source_with_payload(transport_agent, media_session, 0); } static void on_new_send_source(OwrTransportAgent *transport_agent, @@ -981,7 +1056,7 @@ static void on_new_send_source(OwrTransportAgent *transport_agent, remove_existing_send_source_and_payload(transport_agent, old_media_source, media_session); if (new_media_source && old_media_source != new_media_source) - maybe_handle_new_send_source_with_payload(transport_agent, media_session); + maybe_handle_new_send_source_with_payload(transport_agent, media_session, 0); } static gboolean add_session(GHashTable *args) @@ -990,11 +1065,12 @@ static gboolean add_session(GHashTable *args) OwrTransportAgentPrivate *priv; OwrSession *session; guint stream_id; + guint session_id; gboolean rtcp_mux = TRUE; GObject *rtp_session; - GstStateChangeReturn state_change_status; PendingSessionInfo *pending_session_info; guint port; + guint number_sessions; g_return_val_if_fail(args, FALSE); @@ -1005,34 +1081,50 @@ static gboolean add_session(GHashTable *args) g_return_val_if_fail(session, FALSE); priv = transport_agent->priv; - g_mutex_lock(&priv->sessions_lock); + AGENT_SESSIONS_LOCK(transport_agent); if (g_hash_table_find(priv->sessions, (GHRFunc)is_same_session, session)) { g_warning("An already existing media session was added to the transport agent. Action aborted."); g_mutex_unlock(&priv->sessions_lock); goto end; } - g_mutex_unlock(&priv->sessions_lock); + number_sessions = g_hash_table_size(priv->sessions); + AGENT_SESSIONS_UNLOCK(transport_agent); if (OWR_IS_MEDIA_SESSION(session)) g_object_get(OWR_MEDIA_SESSION(session), "rtcp-mux", &rtcp_mux, NULL); - stream_id = nice_agent_add_stream(priv->nice_agent, rtcp_mux ? 1 : 2); - if (!stream_id) { - g_warning("Failed to add media session."); - goto end; + if ((priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) || (number_sessions == 0)) { + stream_id = nice_agent_add_stream(priv->nice_agent, rtcp_mux ? 1 : 2); + if (!stream_id) { + g_warning("Failed to add media session."); + goto end; + } + _owr_session_set_stream_id(OWR_SESSION(session), stream_id); + } else { + GHashTableIter iter; + gpointer key, value; + + g_hash_table_iter_init(&iter, priv->sessions); + g_hash_table_iter_next(&iter, &key, &value); + stream_id = _owr_session_get_stream_id(OWR_SESSION(value)); + _owr_session_set_stream_id(OWR_SESSION(session), stream_id); } - g_mutex_lock(&priv->sessions_lock); - g_hash_table_insert(priv->sessions, GUINT_TO_POINTER(stream_id), session); + session_id = priv->next_session_id++; + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_insert(priv->sessions, GUINT_TO_POINTER(session_id), session); g_object_ref(session); - g_mutex_unlock(&priv->sessions_lock); + AGENT_SESSIONS_UNLOCK(transport_agent); - update_helper_servers(transport_agent, stream_id); + if ((priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) || (number_sessions == 0)) { + update_helper_servers(transport_agent, stream_id); - _owr_session_set_on_remote_candidate(session, - g_cclosure_new_object_swap(G_CALLBACK(on_new_remote_candidate), G_OBJECT(transport_agent))); - _owr_session_set_on_local_candidate_change(session, - g_cclosure_new_object_swap(G_CALLBACK(on_local_candidate_change), G_OBJECT(transport_agent))); + _owr_session_set_on_remote_candidate(session, + g_cclosure_new_object_swap(G_CALLBACK(on_new_remote_candidate), G_OBJECT(transport_agent))); + _owr_session_set_on_local_candidate_change(session, + g_cclosure_new_object_swap(G_CALLBACK(on_local_candidate_change), G_OBJECT(transport_agent))); + } if (OWR_IS_MEDIA_SESSION(session)) { guint send_ssrc = 0; @@ -1043,13 +1135,16 @@ static gboolean add_session(GHashTable *args) _owr_media_session_set_on_send_payload(OWR_MEDIA_SESSION(session), g_cclosure_new_object_swap(G_CALLBACK(on_new_send_payload), G_OBJECT(transport_agent))); - pending_session_info = g_new0(PendingSessionInfo, 1); - prepare_transport_bin_receive_elements(transport_agent, stream_id, rtcp_mux, pending_session_info); - prepare_transport_bin_send_elements(transport_agent, stream_id, rtcp_mux, pending_session_info); - g_mutex_lock(&transport_agent->priv->sessions_lock); - g_hash_table_insert(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id), pending_session_info); - g_mutex_unlock(&transport_agent->priv->sessions_lock); + pending_session_info = g_new0(PendingSessionInfo, 1); + if (((priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) && (number_sessions == 0)) + || (priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE)) { + prepare_transport_bin_receive_elements(transport_agent, session_id, stream_id, rtcp_mux, pending_session_info); + } + prepare_transport_bin_send_elements(transport_agent, session_id, stream_id, rtcp_mux, pending_session_info); + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_insert(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id), pending_session_info); + AGENT_SESSIONS_UNLOCK(transport_agent); g_object_get(session, "send-ssrc", &send_ssrc, "cname", &cname, NULL); if (!send_ssrc || !cname) @@ -1058,8 +1153,8 @@ static gboolean add_session(GHashTable *args) _owr_data_session_set_on_datachannel_added(OWR_DATA_SESSION(session), g_cclosure_new_object_swap(G_CALLBACK(on_new_datachannel), G_OBJECT(transport_agent))); - prepare_transport_bin_data_receive_elements(transport_agent, stream_id); - prepare_transport_bin_data_send_elements(transport_agent, stream_id); + prepare_transport_bin_data_receive_elements(transport_agent, session_id, stream_id); + prepare_transport_bin_data_send_elements(transport_agent, session_id, stream_id); } if (priv->local_max_port > 0) { @@ -1108,13 +1203,14 @@ static gboolean add_session(GHashTable *args) g_object_set(rtp_session, "bandwidth", (gdouble)700000, "rtcp-fraction", (gdouble)100000, "rtcp-min-interval", (guint64)200000000, NULL);*/ - g_object_set_data(rtp_session, "session_id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(rtp_session, "stream-id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(rtp_session, "session-id", GUINT_TO_POINTER(session_id)); + g_object_set_data(rtp_session, "session", session); g_signal_connect_after(rtp_session, "on-sending-rtcp", G_CALLBACK(on_sending_rtcp), transport_agent); g_signal_connect(rtp_session, "on-feedback-rtcp", G_CALLBACK(on_feedback_rtcp), transport_agent); g_signal_connect_after(rtp_session, "on-receiving-rtcp", G_CALLBACK(on_receiving_rtcp), NULL); g_object_unref(rtp_session); - - maybe_handle_new_send_source_with_payload(transport_agent, OWR_MEDIA_SESSION(session)); + maybe_handle_new_send_source_with_payload(transport_agent, OWR_MEDIA_SESSION(session), 0); } if (_owr_session_get_remote_candidates(session)) @@ -1122,8 +1218,6 @@ static gboolean add_session(GHashTable *args) if (_owr_session_get_forced_remote_candidates(session)) on_new_remote_candidate(transport_agent, TRUE, session); - state_change_status = gst_element_set_state(transport_agent->priv->pipeline, GST_STATE_PLAYING); - g_warn_if_fail(state_change_status != GST_STATE_CHANGE_FAILURE); end: g_object_unref(session); @@ -1132,7 +1226,7 @@ static gboolean add_session(GHashTable *args) return FALSE; } -static GstElement *add_nice_element(OwrTransportAgent *transport_agent, guint stream_id, +static GstElement *add_nice_element(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean is_sink, gboolean is_rtcp, GstElement *bin) { GstElement *nice_element = NULL; @@ -1141,8 +1235,18 @@ static GstElement *add_nice_element(OwrTransportAgent *transport_agent, guint st g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), NULL); - element_name = g_strdup_printf("nice-%s-%s-%u", is_rtcp ? "rtcp" : "rtp", is_sink - ? "sink" : "src", stream_id); + if ((transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) && !is_rtcp) + element_name = g_strdup_printf("nice-%s-%s-%u", is_rtcp ? "rtcp" : "rtp", is_sink + ? "sink" : "src", stream_id); + else + element_name = g_strdup_printf("nice-%s-%s-%u-%u", is_rtcp ? "rtcp" : "rtp", is_sink + ? "sink" : "src", session_id, stream_id); + nice_element = gst_bin_get_by_name(GST_BIN(bin), element_name); + if (nice_element && (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE)) { + gst_object_unref(nice_element); + return NULL; + } + nice_element = gst_element_factory_make(is_sink ? "nicesink" : "nicesrc", element_name); g_free(element_name); @@ -1230,7 +1334,7 @@ static void on_dtls_peer_certificate(GstElement *dtls_srtp_bin, GParamSpec *pspe g_free(certificate); } -static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint stream_id, +static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean is_encoder, gboolean is_rtcp, GstElement *bin) { OwrTransportAgentPrivate *priv; @@ -1243,8 +1347,8 @@ static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint s g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), NULL); priv = transport_agent->priv; - element_name = g_strdup_printf("dtls_srtp_%s_%s_%u", is_rtcp ? "rtcp" : "rtp", - is_encoder ? "encoder" : "decoder", stream_id); + element_name = g_strdup_printf("dtls_srtp_%s_%s_%u_%u", is_rtcp ? "rtcp" : "rtp", + is_encoder ? "encoder" : "decoder", session_id, stream_id); dtls_srtp_bin = gst_element_factory_make(is_encoder ? "dtlssrtpenc" : "dtlssrtpdec", element_name); @@ -1253,7 +1357,7 @@ static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint s g_object_set(dtls_srtp_bin, "connection-id", connection_id, NULL); g_free(connection_id); - session = get_session(transport_agent, stream_id); + session = get_session(transport_agent, session_id); if (!is_encoder) { g_object_get(session, "dtls-certificate", &cert, NULL); @@ -1380,10 +1484,10 @@ static void on_bitrate_change(GstElement *scream_queue, guint bitrate, guint ssr _owr_schedule_with_hash_table((GSourceFunc)emit_bitrate_change, args); } -static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, guint stream_id, gboolean rtp, gboolean rtcp) +static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean rtp, gboolean rtcp) { gchar *rtpbin_pad_name, *dtls_srtp_pad_name; - gchar *output_selector_name; + gchar *output_selector_name, *queue_name; gboolean linked_ok; GstPad *sink_pad, *src_pad; GstElement *output_selector; @@ -1397,7 +1501,8 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); send_bin_info = g_hash_table_lookup(transport_agent->priv->send_bins, GINT_TO_POINTER(stream_id)); - g_return_if_fail(send_bin_info); + if (!send_bin_info) + return; dtls_srtp_bin_rtp = send_bin_info->dtls_srtp_bin_rtp; dtls_srtp_bin_rtcp = send_bin_info->dtls_srtp_bin_rtcp; @@ -1406,9 +1511,11 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g g_return_if_fail(GST_IS_ELEMENT(dtls_srtp_bin_rtp)); g_return_if_fail(!dtls_srtp_bin_rtcp || GST_IS_ELEMENT(dtls_srtp_bin_rtcp)); - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, stream_id)); + media_session = OWR_MEDIA_SESSION(get_session_unlocked(transport_agent, session_id)); - scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), "screamqueue"); + queue_name = g_strdup_printf("screamqueue-%u-%u", session_id, stream_id); + scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), queue_name); + g_free(queue_name); g_assert(scream_queue); g_signal_connect_object(scream_queue, "on-bitrate-change", G_CALLBACK(on_bitrate_change), media_session, 0); @@ -1416,8 +1523,8 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g /* RTP */ if (rtp) { - rtpbin_pad_name = g_strdup_printf("send_rtp_src_%u", stream_id); - dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", stream_id); + rtpbin_pad_name = g_strdup_printf("send_rtp_src_%u", session_id); + dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", session_id + stream_id); sink_pad = gst_element_get_static_pad(scream_queue, "sink"); g_assert(GST_IS_PAD(sink_pad)); @@ -1432,23 +1539,28 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g g_free(dtls_srtp_pad_name); } + gst_object_unref(scream_queue); + /* RTCP */ - if (rtcp && !send_bin_info->linked_rtcp) { - output_selector_name = g_strdup_printf("rtcp_output_selector_%u", stream_id); + output_selector_name = g_strdup_printf("rtcp-output-selector-%u-%u", session_id, stream_id); + output_selector = gst_bin_get_by_name(GST_BIN(send_output_bin), output_selector_name); + if (output_selector) { + gst_object_unref(output_selector); + } else if (rtcp) { output_selector = gst_element_factory_make("output-selector", output_selector_name); - g_free(output_selector_name); g_object_set(output_selector, "pad-negotiation-mode", 0, NULL); gst_bin_add(GST_BIN(send_output_bin), output_selector); gst_element_sync_state_with_parent(output_selector); - rtpbin_pad_name = g_strdup_printf("send_rtcp_src_%u", stream_id); - dtls_srtp_pad_name = g_strdup_printf("rtcp_sink_%u", stream_id); + rtpbin_pad_name = g_strdup_printf("send_rtcp_src_%u", session_id); + dtls_srtp_pad_name = g_strdup_printf("rtcp_sink_%u", session_id); /* RTCP muxing */ - sink_pad = gst_element_get_request_pad(dtls_srtp_bin_rtp, dtls_srtp_pad_name); - g_assert(GST_IS_PAD(sink_pad)); src_pad = gst_element_get_request_pad(output_selector, "src_%u"); g_assert(GST_IS_PAD(src_pad)); + sink_pad = gst_element_get_request_pad(dtls_srtp_bin_rtp, dtls_srtp_pad_name); + g_assert(GST_IS_PAD(sink_pad)); + linked_ok = gst_pad_link(src_pad, sink_pad) == GST_PAD_LINK_OK; g_warn_if_fail(linked_ok); g_object_set(output_selector, "active-pad", src_pad, NULL); @@ -1486,73 +1598,130 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g send_bin_info->linked_rtcp = TRUE; } + g_free(output_selector_name); } static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_agent, - guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) + guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) { - GstElement *nice_element, *dtls_srtp_bin_rtp, *dtls_srtp_bin_rtcp = NULL; + GstElement *nice_element, *dtls_srtp_bin_rtp = NULL, *dtls_srtp_bin_rtcp = NULL; GstElement *scream_queue = NULL; gboolean linked_ok, synced_ok; - GstElement *send_output_bin; + GstElement *send_output_bin = NULL; SendBinInfo *send_bin_info; - gchar *bin_name, *dtls_srtp_pad_name; + gchar *bin_name, *dtls_srtp_pad_name = NULL, *queue_name; OwrMediaSession *media_session; AgentAndSessionIdPair *agent_and_session_id_pair; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); bin_name = g_strdup_printf("send-output-bin-%u", stream_id); - send_output_bin = gst_bin_new(bin_name); - g_free(bin_name); - - if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_output_bin)) { - GST_ERROR("Failed to add send-output-bin-%u to parent bin", stream_id); - return; + if (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + send_output_bin = gst_bin_get_by_name(GST_BIN(transport_agent->priv->transport_bin), bin_name); } - if (!gst_element_sync_state_with_parent(send_output_bin)) { - GST_ERROR("Failed to sync send-output-bin-%u state with parent bin", stream_id); - return; + if (!send_output_bin) { + send_output_bin = gst_bin_new(bin_name); + + if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_output_bin)) { + GST_ERROR("Failed to add %s to parent bin", bin_name); + g_free(bin_name); + return; + } + + if (!gst_element_sync_state_with_parent(send_output_bin)) { + GST_ERROR("Failed to sync %s state with parent bin", bin_name); + g_free(bin_name); + return; + } } - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, stream_id)); - scream_queue = gst_element_factory_make("screamqueue", "screamqueue"); + g_free(bin_name); + + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + queue_name = g_strdup_printf("screamqueue-%u-%u", session_id, stream_id); + scream_queue = gst_element_factory_make("screamqueue", queue_name); + g_free(queue_name); g_assert(scream_queue); g_object_set(scream_queue, "scream-controller-id", transport_agent->priv->agent_id, NULL); g_signal_connect(scream_queue, "on-payload-adaptation-request", (GCallback)on_payload_adaptation_request, media_session); gst_bin_add(GST_BIN(send_output_bin), scream_queue); - pending_session_info->nice_sink_rtp = nice_element = add_nice_element(transport_agent, stream_id, TRUE, FALSE, send_output_bin); - pending_session_info->dtls_enc_rtp = dtls_srtp_bin_rtp = add_dtls_srtp_bin(transport_agent, stream_id, TRUE, FALSE, send_output_bin); - linked_ok = gst_element_link(dtls_srtp_bin_rtp, nice_element); - g_warn_if_fail(linked_ok); + nice_element = add_nice_element(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); + // In MAX_BUNDLE case we need only one dtlssrtpenc and nicesink. + if (nice_element) { +#ifdef TEST_RTX + GstElement* identity = gst_element_factory_make("identity", NULL); + g_object_set(identity, "drop-probability", 0.01, NULL); - agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); - agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; - g_signal_connect_data(dtls_srtp_bin_rtp, "on-key-set", G_CALLBACK(on_dtls_enc_key_set), agent_and_session_id_pair, (GClosureNotify) g_free, 0); + gst_bin_add(GST_BIN(send_output_bin), identity); +#endif + pending_session_info->nice_sink_rtp = nice_element; + pending_session_info->dtls_enc_rtp = dtls_srtp_bin_rtp = add_dtls_srtp_bin(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); +#ifdef TEST_RTX + linked_ok = gst_element_link(identity, nice_element); + linked_ok &= gst_element_link(dtls_srtp_bin_rtp, identity); +#else + linked_ok = gst_element_link(dtls_srtp_bin_rtp, nice_element); +#endif + g_warn_if_fail(linked_ok); - dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", stream_id); - linked_ok = gst_element_link_pads(scream_queue, "src", dtls_srtp_bin_rtp, dtls_srtp_pad_name); - g_free(dtls_srtp_pad_name); - g_warn_if_fail(linked_ok); + agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); + agent_and_session_id_pair->transport_agent = transport_agent; + agent_and_session_id_pair->session_id = session_id; + g_signal_connect_data(dtls_srtp_bin_rtp, "on-key-set", G_CALLBACK(on_dtls_enc_key_set), agent_and_session_id_pair, (GClosureNotify) g_free, 0); + + dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", stream_id); + } else { + GSList *sessions = get_sessions_from_stream_id(transport_agent, stream_id); + guint previous_session_id = 0; + GSList *walk; + // Look for the session for which the global encoder was created. + for (walk = sessions; walk && previous_session_id == 0; walk = g_slist_next(walk)) { + OwrMediaSession *current = OWR_MEDIA_SESSION(walk->data); + guint current_session_id; + + if (!OWR_IS_MEDIA_SESSION(current)) + continue; + + current_session_id = get_session_id(transport_agent, OWR_SESSION(current)); + if (current_session_id != session_id) + previous_session_id = current_session_id; + } + g_slist_free_full(sessions, g_object_unref); + if (previous_session_id > 0) { + bin_name = g_strdup_printf("dtls_srtp_rtp_encoder_%u_%u", previous_session_id, stream_id); + dtls_srtp_bin_rtp = gst_bin_get_by_name(GST_BIN(send_output_bin), bin_name); + g_free(bin_name); + dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", session_id); + } + } + + if (dtls_srtp_bin_rtp && dtls_srtp_pad_name) { + linked_ok = gst_element_link_pads(scream_queue, "src", dtls_srtp_bin_rtp, dtls_srtp_pad_name); + g_warn_if_fail(linked_ok); + g_free(dtls_srtp_pad_name); + } + + if (nice_element) { + synced_ok = gst_element_sync_state_with_parent(nice_element); + g_warn_if_fail(synced_ok); + } - synced_ok = gst_element_sync_state_with_parent(nice_element); - g_warn_if_fail(synced_ok); synced_ok = gst_element_sync_state_with_parent(scream_queue); g_warn_if_fail(synced_ok); if (!rtcp_mux) { - pending_session_info->nice_sink_rtcp = nice_element = add_nice_element(transport_agent, stream_id, TRUE, TRUE, send_output_bin); - pending_session_info->dtls_enc_rtcp = dtls_srtp_bin_rtcp = add_dtls_srtp_bin(transport_agent, stream_id, TRUE, TRUE, send_output_bin); + nice_element = add_nice_element(transport_agent, session_id, stream_id, TRUE, TRUE, send_output_bin); + pending_session_info->nice_sink_rtcp = nice_element; + pending_session_info->dtls_enc_rtcp = dtls_srtp_bin_rtcp = add_dtls_srtp_bin(transport_agent, session_id, stream_id, TRUE, TRUE, send_output_bin); linked_ok = gst_element_link(dtls_srtp_bin_rtcp, nice_element); g_warn_if_fail(linked_ok); agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; + agent_and_session_id_pair->session_id = session_id; g_signal_connect_data(dtls_srtp_bin_rtp, "on-key-set", G_CALLBACK(on_dtls_enc_key_set), agent_and_session_id_pair, (GClosureNotify) g_free, 0); synced_ok = gst_element_sync_state_with_parent(nice_element); @@ -1568,7 +1737,7 @@ static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_age g_hash_table_insert(transport_agent->priv->send_bins, GINT_TO_POINTER(stream_id), send_bin_info); } -static GstPadProbeReturn nice_src_pad_block(GstPad *pad, GstPadProbeInfo *info, AgentAndSessionIdPair *data) +static GstPadProbeReturn nice_src_pad_block(GstPad *pad, GstPadProbeInfo *info, gpointer data) { OWR_UNUSED(pad); OWR_UNUSED(info); @@ -1578,70 +1747,77 @@ static GstPadProbeReturn nice_src_pad_block(GstPad *pad, GstPadProbeInfo *info, } static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_agent, - guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) + guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) { GstElement *nice_element, *dtls_srtp_bin, *funnel; GstPad *rtp_src_pad, *rtcp_src_pad, *rtp_sink_pad; GstPad *nice_src_pad; - gchar *rtpbin_pad_name; gboolean linked_ok, synced_ok; GstElement *receive_input_bin; ScreamRx *scream_rx; + gchar *bin_name; + gchar *rtp_src_pad_name; + gchar *rtpbin_pad_name; #ifdef TEST_RTX GstElement *identity; #endif - gchar *bin_name; AgentAndSessionIdPair *agent_and_session_id_pair; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - bin_name = g_strdup_printf("receive-input-bin-%u", stream_id); + if (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + bin_name = g_strdup_printf("receive-input-bin-%u", session_id); + } else { + bin_name = g_strdup_printf("receive-input-bin-%u", stream_id); + } + receive_input_bin = gst_bin_new(bin_name); - g_free(bin_name); if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), receive_input_bin)) { - GST_ERROR("Failed to add receive-input-bin-%u to parent bin", stream_id); + GST_ERROR("Failed to add %s to parent bin", bin_name); + g_free(bin_name); return; } if (!gst_element_sync_state_with_parent(receive_input_bin)) { - GST_ERROR("Failed to sync receive-input-bin-%u state with parent bin", stream_id); + GST_ERROR("Failed to sync %s state with parent bin", bin_name); + g_free(bin_name); return; } + g_free(bin_name); - pending_session_info->nice_src_rtp = nice_element = add_nice_element(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); - pending_session_info->dtls_dec_rtp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); + pending_session_info->nice_src_rtp = nice_element = add_nice_element(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); + pending_session_info->dtls_dec_rtp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); - agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); - agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; nice_src_pad = gst_element_get_static_pad(nice_element, "src"); - pending_session_info->nice_src_block_rtp = gst_pad_add_probe(nice_src_pad, GST_PAD_PROBE_TYPE_BLOCK | GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST, (GstPadProbeCallback)nice_src_pad_block, agent_and_session_id_pair, (GDestroyNotify) g_free); + pending_session_info->nice_src_block_rtp = gst_pad_add_probe(nice_src_pad, GST_PAD_PROBE_TYPE_BLOCK | GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST, (GstPadProbeCallback)nice_src_pad_block, NULL, NULL); gst_object_unref(nice_src_pad); synced_ok = gst_element_sync_state_with_parent(nice_element); g_warn_if_fail(synced_ok); - rtp_src_pad = gst_element_get_static_pad(dtls_srtp_bin, "rtp_src"); - ghost_pad_and_add_to_bin(rtp_src_pad, receive_input_bin, "rtp_src"); + rtp_src_pad_name = g_strdup("rtp_src"); + rtp_src_pad = gst_element_get_static_pad(dtls_srtp_bin, rtp_src_pad_name); + ghost_pad_and_add_to_bin(rtp_src_pad, receive_input_bin, rtp_src_pad_name); gst_object_unref(rtp_src_pad); rtpbin_pad_name = g_strdup_printf("recv_rtp_sink_%u", stream_id); #ifndef TEST_RTX - linked_ok = gst_element_link_pads(receive_input_bin, "rtp_src", transport_agent->priv->rtpbin, + linked_ok = gst_element_link_pads(receive_input_bin, rtp_src_pad_name, transport_agent->priv->rtpbin, rtpbin_pad_name); #else identity = gst_element_factory_make("identity", NULL); g_object_set(identity, "drop-probability", 0.01, NULL); gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), identity); - gst_element_link_pads(receive_input_bin, "rtp_src", identity, "sink"); + gst_element_link_pads(receive_input_bin, rtp_src_pad_name, identity, "sink"); linked_ok = gst_element_link_pads(identity, "src", transport_agent->priv->rtpbin, rtpbin_pad_name); gst_element_sync_state_with_parent(identity); #endif g_warn_if_fail(linked_ok); g_free(rtpbin_pad_name); + g_free(rtp_src_pad_name); if (!rtcp_mux) { funnel = gst_element_factory_make("funnel", NULL); @@ -1650,12 +1826,12 @@ static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_ linked_ok = gst_element_link_pads(dtls_srtp_bin, "rtcp_src", funnel, "sink_0"); g_warn_if_fail(linked_ok); - pending_session_info->nice_src_rtcp = nice_element = add_nice_element(transport_agent, stream_id, FALSE, TRUE, receive_input_bin); - pending_session_info->dtls_dec_rtcp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, FALSE, TRUE, receive_input_bin); + pending_session_info->nice_src_rtcp = nice_element = add_nice_element(transport_agent, session_id, stream_id, FALSE, TRUE, receive_input_bin); + pending_session_info->dtls_dec_rtcp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, FALSE, TRUE, receive_input_bin); agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; + agent_and_session_id_pair->session_id = session_id; nice_src_pad = gst_element_get_static_pad(nice_element, "src"); pending_session_info->nice_src_block_rtcp = gst_pad_add_probe(nice_src_pad, GST_PAD_PROBE_TYPE_BLOCK | GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST, (GstPadProbeCallback)nice_src_pad_block, agent_and_session_id_pair, (GDestroyNotify) g_free); gst_object_unref(nice_src_pad); @@ -1691,7 +1867,8 @@ static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_ scream_rx = g_new0(ScreamRx, 1); scream_rx->rtx_pt = -2; /* unknown */ scream_rx->transport_agent = transport_agent; - scream_rx->session_id = stream_id; + scream_rx->stream_id = stream_id; + scream_rx->session_id = session_id; scream_rx->adapt = TRUE; /* Always initiates to TRUE. Sets to TRUE or FALSE in probe_rtp_info */ gst_pad_add_probe(rtp_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)probe_rtp_info, scream_rx, g_free); @@ -1700,7 +1877,7 @@ static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_ static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *transport_agent, - guint stream_id) + guint session_id, guint stream_id) { OwrTransportAgentPrivate *priv; GstElement *nice_element, *dtls_srtp_bin, *sctpdec; @@ -1729,12 +1906,12 @@ static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *trans return; } - nice_element = add_nice_element(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); - dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); + nice_element = add_nice_element(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); + dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); sctpdec = _owr_data_session_create_decoder(data_session); g_return_if_fail(nice_element && dtls_srtp_bin && sctpdec); - g_object_set_data(G_OBJECT(sctpdec), "session-id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(G_OBJECT(sctpdec), "stream_id", GUINT_TO_POINTER(stream_id)); gst_bin_add(GST_BIN(receive_input_bin), sctpdec); g_signal_connect(sctpdec, "pad-added", (GCallback)sctpdec_pad_added, transport_agent); g_signal_connect(sctpdec, "pad-removed", (GCallback)sctpdec_pad_removed, transport_agent); @@ -1750,7 +1927,7 @@ static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *trans } static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transport_agent, - guint stream_id) + guint session_id, guint stream_id) { OwrTransportAgentPrivate *priv; GstElement *nice_element, *dtls_srtp_bin, *sctpenc, *send_output_bin; @@ -1764,23 +1941,25 @@ static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transpor name = g_strdup_printf("send-output-bin-%u", stream_id); send_output_bin = gst_bin_new(name); - g_free(name); if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_output_bin)) { - GST_ERROR("Failed to add send-output-bin-%u to parent bin", stream_id); + GST_ERROR("Failed to add %s to parent bin", name); + g_free(name); return; } if (!gst_element_sync_state_with_parent(send_output_bin)) { - GST_ERROR("Failed to sync send-output-bin-%u to parent bin", stream_id); + GST_ERROR("Failed to sync %s to parent bin", name); + g_free(name); return; } + g_free(name); - nice_element = add_nice_element(transport_agent, stream_id, TRUE, FALSE, send_output_bin); - dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, TRUE, FALSE, send_output_bin); + nice_element = add_nice_element(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); + dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); sctpenc = _owr_data_session_create_encoder(data_session); g_warn_if_fail(sctpenc); - g_object_set_data(G_OBJECT(sctpenc), "session-id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(G_OBJECT(sctpenc), "stream_id", GUINT_TO_POINTER(stream_id)); gst_bin_add(GST_BIN(send_output_bin), sctpenc); g_signal_connect(sctpenc, "sctp-association-established", G_CALLBACK(on_sctp_association_established), transport_agent); @@ -1805,7 +1984,7 @@ static void set_send_ssrc_and_cname(OwrTransportAgent *transport_agent, OwrMedia g_return_if_fail(transport_agent); g_return_if_fail(media_session); - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); g_signal_emit_by_name(transport_agent->priv->rtpbin, "get-internal-session", stream_id, &session); g_warn_if_fail(session); g_object_get(session, "internal-ssrc", &send_ssrc, "sdes", &sdes, NULL); @@ -1819,11 +1998,12 @@ static gboolean emit_new_candidate(GHashTable *args) { OwrTransportAgent *transport_agent; OwrTransportAgentPrivate *priv; - OwrSession *session; NiceCandidate *nice_candidate; OwrCandidate *owr_candidate; gchar *ufrag = NULL, *password = NULL; gboolean got_credentials; + GSList *sessions; + GSList *walk; transport_agent = OWR_TRANSPORT_AGENT(g_hash_table_lookup(args, "transport_agent")); g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), FALSE); @@ -1831,33 +2011,36 @@ static gboolean emit_new_candidate(GHashTable *args) nice_candidate = (NiceCandidate *)g_hash_table_lookup(args, "nice_candidate"); g_return_val_if_fail(nice_candidate, FALSE); - session = get_session(transport_agent, nice_candidate->stream_id); - g_return_val_if_fail(OWR_IS_SESSION(session), FALSE); - if (!nice_candidate->username || !nice_candidate->password) { - got_credentials = nice_agent_get_local_credentials(priv->nice_agent, - nice_candidate->stream_id, &ufrag, &password); - g_warn_if_fail(got_credentials); + sessions = get_sessions_from_stream_id(transport_agent, nice_candidate->stream_id); + for (walk = sessions; walk; walk = g_slist_next(walk)) { + OwrSession *session = OWR_SESSION(walk->data); + if (!nice_candidate->username || !nice_candidate->password) { + got_credentials = nice_agent_get_local_credentials(priv->nice_agent, + nice_candidate->stream_id, &ufrag, &password); + g_warn_if_fail(got_credentials); + + if (!nice_candidate->username) + nice_candidate->username = ufrag; + else + g_free(ufrag); + + if (!nice_candidate->password) + nice_candidate->password = password; + else + g_free(password); + } - if (!nice_candidate->username) - nice_candidate->username = ufrag; - else - g_free(ufrag); + owr_candidate = _owr_candidate_new_from_nice_candidate(nice_candidate); + if (!owr_candidate) + continue; - if (!nice_candidate->password) - nice_candidate->password = password; - else - g_free(password); + g_signal_emit_by_name(session, "on-new-candidate", owr_candidate); + g_object_unref(owr_candidate); } - owr_candidate = _owr_candidate_new_from_nice_candidate(nice_candidate); - g_return_val_if_fail(owr_candidate, FALSE); + g_slist_free_full(sessions, g_object_unref); nice_candidate_free(nice_candidate); - - g_signal_emit_by_name(session, "on-new-candidate", owr_candidate); - g_object_unref(owr_candidate); - - g_object_unref(session); g_hash_table_destroy(args); g_object_unref(transport_agent); @@ -1889,56 +2072,56 @@ static gboolean emit_candidate_gathering_done(GHashTable *args) OwrCandidate *local_candidate, *remote_candidate; guint stream_id; int i; + GSList *sessions; + GSList *walk; transport_agent = g_hash_table_lookup(args, "transport-agent"); - session = g_hash_table_lookup(args, "session"); - g_signal_emit_by_name(session, "on-candidate-gathering-done", NULL); + stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "stream-id")); + + sessions = get_sessions_from_stream_id(transport_agent, stream_id); + for (walk = sessions; walk; walk = g_slist_next(walk)) { + session = OWR_SESSION(walk->data); - stream_id = get_stream_id(transport_agent, session); - g_return_val_if_fail(stream_id, FALSE); + g_signal_emit_by_name(session, "on-candidate-gathering-done", NULL); - for (i = 0; i < OWR_COMPONENT_MAX; i++) { - _owr_session_get_candidate_pair(session, i, &local_candidate, &remote_candidate); + for (i = 0; i < OWR_COMPONENT_MAX; i++) { + _owr_session_get_candidate_pair(session, i, &local_candidate, &remote_candidate); - if (local_candidate && remote_candidate) { - gchar *lfoundation = NULL, *rfoundation = NULL; + if (local_candidate && remote_candidate) { + gchar *lfoundation = NULL, *rfoundation = NULL; - g_object_get(local_candidate, "foundation", &lfoundation, NULL); - g_object_get(remote_candidate, "foundation", &rfoundation, NULL); + g_object_get(local_candidate, "foundation", &lfoundation, NULL); + g_object_get(remote_candidate, "foundation", &rfoundation, NULL); - if (lfoundation && rfoundation) { - GST_DEBUG_OBJECT(transport_agent, "Forcing pair %s:%s on stream %u", - lfoundation, rfoundation, stream_id); - nice_agent_set_selected_pair(transport_agent->priv->nice_agent, - stream_id, i, lfoundation, rfoundation); - } else - g_assert_not_reached(); + if (lfoundation && rfoundation) { + GST_DEBUG_OBJECT(transport_agent, "Forcing pair %s:%s on stream %u", + lfoundation, rfoundation, stream_id); + nice_agent_set_selected_pair(transport_agent->priv->nice_agent, + stream_id, i, lfoundation, rfoundation); + } else + g_assert_not_reached(); - g_free(lfoundation); - g_free(rfoundation); + g_free(lfoundation); + g_free(rfoundation); + } } } - + g_slist_free_full(sessions, g_object_unref); g_hash_table_destroy(args); - g_object_unref(session); return FALSE; } static void on_candidate_gathering_done(NiceAgent *nice_agent, guint stream_id, OwrTransportAgent *transport_agent) { - OwrSession *session; GHashTable *args; g_return_if_fail(nice_agent); g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - session = get_session(transport_agent, stream_id); - g_return_if_fail(OWR_IS_SESSION(session)); - - args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(session)); + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(transport_agent)); g_hash_table_insert(args, "transport-agent", transport_agent); - g_hash_table_insert(args, "session", session); + g_hash_table_insert(args, "stream-id", GUINT_TO_POINTER(stream_id)); _owr_schedule_with_hash_table((GSourceFunc)emit_candidate_gathering_done, args); @@ -1946,20 +2129,26 @@ static void on_candidate_gathering_done(NiceAgent *nice_agent, guint stream_id, static gboolean emit_ice_state_changed(GHashTable *args) { - OwrSession *session; - guint session_id; + OwrTransportAgent *transport_agent; + guint stream_id; OwrComponentType component_type; OwrIceState state; + GSList *sessions; + GSList *walk; - session = g_hash_table_lookup(args, "session"); - session_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "session-id")); + transport_agent = g_hash_table_lookup(args, "transport-agent"); + stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "stream-id")); component_type = GPOINTER_TO_UINT(g_hash_table_lookup(args, "component-type")); state = GPOINTER_TO_UINT(g_hash_table_lookup(args, "ice-state")); - _owr_session_emit_ice_state_changed(session, session_id, component_type, state); + sessions = get_sessions_from_stream_id(transport_agent, stream_id); + for (walk = sessions; walk; walk = g_slist_next(walk)) { + OwrSession *session = OWR_SESSION(walk->data); + _owr_session_emit_ice_state_changed(session, stream_id, component_type, state); + } + g_slist_free_full(sessions, g_object_unref); g_hash_table_destroy(args); - g_object_unref(session); return FALSE; } @@ -1967,18 +2156,14 @@ static gboolean emit_ice_state_changed(GHashTable *args) static void on_component_state_changed(NiceAgent *nice_agent, guint stream_id, guint component_id, OwrIceState state, OwrTransportAgent *transport_agent) { - OwrSession *session; GHashTable *args; g_return_if_fail(nice_agent); g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - session = get_session(transport_agent, stream_id); - g_return_if_fail(OWR_IS_SESSION(session)); - - args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(session)); - g_hash_table_insert(args, "session", session); - g_hash_table_insert(args, "session-id", GUINT_TO_POINTER(stream_id)); + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(transport_agent)); + g_hash_table_insert(args, "transport-agent", transport_agent); + g_hash_table_insert(args, "stream-id", GUINT_TO_POINTER(stream_id)); g_hash_table_insert(args, "component-type", GUINT_TO_POINTER(component_id)); g_hash_table_insert(args, "ice-state", GUINT_TO_POINTER(state)); @@ -1991,8 +2176,8 @@ static void on_new_selected_pair(NiceAgent *nice_agent, NiceCandidate *lcandidate, NiceCandidate *rcandidate, OwrTransportAgent *transport_agent) { - OwrSession *session; - PendingSessionInfo *pending_session_info; + GSList *sessions; + GSList *walk; OWR_UNUSED(nice_agent); OWR_UNUSED(lcandidate); @@ -2000,70 +2185,78 @@ static void on_new_selected_pair(NiceAgent *nice_agent, g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - session = get_session(transport_agent, stream_id); - g_return_if_fail(OWR_IS_SESSION(session)); - - g_mutex_lock(&transport_agent->priv->sessions_lock); - pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id)); - if (pending_session_info) { - if (component_id == NICE_COMPONENT_TYPE_RTP && pending_session_info->nice_src_block_rtp) { - GstPad *pad; - gboolean sync_ok, link_ok; - - link_ok = gst_element_link(pending_session_info->nice_src_rtp, pending_session_info->dtls_dec_rtp); - g_warn_if_fail(link_ok); - - gst_element_set_locked_state(pending_session_info->dtls_dec_rtp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtp); - g_warn_if_fail(sync_ok); - gst_element_set_locked_state(pending_session_info->dtls_enc_rtp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtp); - g_warn_if_fail(sync_ok); - - pad = gst_element_get_static_pad(pending_session_info->nice_src_rtp, "src"); - gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtp); - gst_object_unref(pad); - pending_session_info->nice_src_block_rtp = 0; - } + sessions = get_sessions_from_stream_id(transport_agent, stream_id); + for (walk = sessions; walk; walk = g_slist_next(walk)) { + OwrSession *session = OWR_SESSION(walk->data); + guint session_id = get_session_id(transport_agent, session); + PendingSessionInfo *pending_session_info; + + AGENT_SESSIONS_LOCK(transport_agent); + pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id)); + if (pending_session_info) { + if (component_id == NICE_COMPONENT_TYPE_RTP && pending_session_info->nice_src_block_rtp) { + GstPad *pad; + gboolean sync_ok, link_ok; + + link_ok = gst_element_link(pending_session_info->nice_src_rtp, pending_session_info->dtls_dec_rtp); + g_warn_if_fail(link_ok); + + gst_element_set_locked_state(pending_session_info->dtls_dec_rtp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtp); + g_warn_if_fail(sync_ok); + gst_element_set_locked_state(pending_session_info->dtls_enc_rtp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtp); + g_warn_if_fail(sync_ok); + + pad = gst_element_get_static_pad(pending_session_info->nice_src_rtp, "src"); + gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtp); + gst_object_unref(pad); + pending_session_info->nice_src_block_rtp = 0; + } - /* when doing standalone RTCP we unblock the RTCP bin and sync its state whenever the first component of - * the stream is connected. This is because the actual RTCP connection might be established - * much later (or even never). Also see link_rtpbin_to_send_output_bin. - */ - if (pending_session_info->nice_src_block_rtcp) { - GstPad *pad; - gboolean sync_ok, link_ok; - - link_ok = gst_element_link(pending_session_info->nice_src_rtcp, pending_session_info->dtls_dec_rtcp); - g_warn_if_fail(link_ok); - - gst_element_set_locked_state(pending_session_info->dtls_dec_rtcp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtcp); - g_warn_if_fail(sync_ok); - gst_element_set_locked_state(pending_session_info->dtls_enc_rtcp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtcp); - g_warn_if_fail(sync_ok); - - pad = gst_element_get_static_pad(pending_session_info->nice_src_rtcp, "src"); - gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtcp); - gst_object_unref(pad); - pending_session_info->nice_src_block_rtcp = 0; + /* when doing standalone RTCP we unblock the RTCP bin and sync its state whenever the first component of + * the stream is connected. This is because the actual RTCP connection might be established + * much later (or even never). Also see link_rtpbin_to_send_output_bin. + */ + if (pending_session_info->nice_src_block_rtcp) { + GstPad *pad; + gboolean sync_ok, link_ok; + + link_ok = gst_element_link(pending_session_info->nice_src_rtcp, pending_session_info->dtls_dec_rtcp); + g_warn_if_fail(link_ok); + + gst_element_set_locked_state(pending_session_info->dtls_dec_rtcp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtcp); + g_warn_if_fail(sync_ok); + gst_element_set_locked_state(pending_session_info->dtls_enc_rtcp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtcp); + g_warn_if_fail(sync_ok); + + pad = gst_element_get_static_pad(pending_session_info->nice_src_rtcp, "src"); + gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtcp); + gst_object_unref(pad); + pending_session_info->nice_src_block_rtcp = 0; + } } + AGENT_SESSIONS_UNLOCK(transport_agent); } - g_mutex_unlock(&transport_agent->priv->sessions_lock); + g_slist_free_full(sessions, g_object_unref); + } static gboolean maybe_handle_new_send_source_with_payload_from_main_thread(GHashTable *args) { OwrTransportAgent *transport_agent; OwrMediaSession *session; + guint process_bundled_sessions; session = OWR_MEDIA_SESSION(g_hash_table_lookup(args, "session")); g_return_val_if_fail(OWR_IS_MEDIA_SESSION(session), FALSE); transport_agent = OWR_TRANSPORT_AGENT(g_hash_table_lookup(args, "transport_agent")); g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), FALSE); + process_bundled_sessions = GPOINTER_TO_UINT(g_hash_table_lookup(args, "process_bundled_sessions")); - maybe_handle_new_send_source_with_payload(transport_agent, session); + maybe_handle_new_send_source_with_payload(transport_agent, session, process_bundled_sessions); g_hash_table_destroy(args); g_object_unref(session); @@ -2077,80 +2270,153 @@ on_dtls_enc_key_set(GstElement *dtls_srtp_enc, AgentAndSessionIdPair *data) { OwrTransportAgent *transport_agent = data->transport_agent; OwrSession *session; - guint stream_id = data->session_id; + guint session_id = data->session_id; PendingSessionInfo *pending_session_info; - session = get_session(transport_agent, stream_id); + session = get_session(transport_agent, session_id); g_return_if_fail(session); /* Once we have the key, the DTLS handshake is done and we can start sending data here. Note * that we only wait for the DTLS handshake to be completed for the RTP component. */ - g_mutex_lock(&transport_agent->priv->sessions_lock); - pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id)); + AGENT_SESSIONS_LOCK(transport_agent); + pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id)); /* FIXME: What to do about RTCP? It's not guaranteed to ever be enabled if * RTCP muxing is used but the usage wasn't known beforehand */ if (pending_session_info && dtls_srtp_enc == pending_session_info->dtls_enc_rtp) { GHashTable *args; + gboolean process_bundled_sessions = transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE; - g_hash_table_remove(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id)); + if (!process_bundled_sessions) + g_hash_table_remove(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id)); args = g_hash_table_new(g_str_hash, g_str_equal); g_hash_table_insert(args, "session", session); g_hash_table_insert(args, "transport_agent", g_object_ref(transport_agent)); + g_hash_table_insert(args, "process_bundled_sessions", GUINT_TO_POINTER(process_bundled_sessions)); _owr_schedule_with_hash_table((GSourceFunc)maybe_handle_new_send_source_with_payload_from_main_thread, args); } - g_mutex_unlock(&transport_agent->priv->sessions_lock); + AGENT_SESSIONS_UNLOCK(transport_agent); g_object_unref(session); } -static guint get_stream_id(OwrTransportAgent *transport_agent, OwrSession *session) +static guint get_session_id_unlocked(OwrTransportAgent *transport_agent, OwrSession *session) { GHashTableIter iter; OwrSession *s; - gpointer stream_id = GUINT_TO_POINTER(0); + gpointer session_id = GUINT_TO_POINTER(0); - g_mutex_lock(&transport_agent->priv->sessions_lock); g_hash_table_iter_init(&iter, transport_agent->priv->sessions); - while (g_hash_table_iter_next(&iter, &stream_id, (gpointer)&s)) { + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&s)) { if (s == session) { - g_mutex_unlock(&transport_agent->priv->sessions_lock); - return GPOINTER_TO_UINT(stream_id); + return GPOINTER_TO_UINT(session_id); } } - g_mutex_unlock(&transport_agent->priv->sessions_lock); - g_warn_if_reached(); return 0; } -static OwrSession * get_session(OwrTransportAgent *transport_agent, guint stream_id) +static guint get_session_id(OwrTransportAgent *transport_agent, OwrSession *session) +{ + guint session_id = 0; + + AGENT_SESSIONS_LOCK(transport_agent); + session_id = get_session_id_unlocked(transport_agent, session); + AGENT_SESSIONS_UNLOCK(transport_agent); + + return session_id; +} + +static OwrSession * get_session_unlocked(OwrTransportAgent *transport_agent, guint session_id) { OwrSession *s; - g_mutex_lock(&transport_agent->priv->sessions_lock); - s = OWR_SESSION(g_hash_table_lookup(transport_agent->priv->sessions, GUINT_TO_POINTER(stream_id))); + s = OWR_SESSION(g_hash_table_lookup(transport_agent->priv->sessions, GUINT_TO_POINTER(session_id))); if (s) g_object_ref(s); - g_mutex_unlock(&transport_agent->priv->sessions_lock); return s; } -static void update_flip_method(OwrPayload *payload, GParamSpec *pspec, GstElement *flip) +static OwrSession * get_session(OwrTransportAgent *transport_agent, guint session_id) { - guint rotation = 0; - gboolean mirror = FALSE; - gint flip_method; + OwrSession *s; - g_return_if_fail(OWR_IS_VIDEO_PAYLOAD(payload)); - g_return_if_fail(G_IS_PARAM_SPEC(pspec) || !pspec); - g_return_if_fail(GST_IS_ELEMENT(flip)); + AGENT_SESSIONS_LOCK(transport_agent); + s = get_session_unlocked(transport_agent, session_id); + AGENT_SESSIONS_UNLOCK(transport_agent); - g_object_get(payload, "rotation", &rotation, "mirror", &mirror, NULL); - flip_method = _owr_rotation_and_mirror_to_video_flip_method(rotation, mirror); - g_object_set(flip, "method", flip_method, NULL); + return s; +} + +/* FIXME: for bundling it must return a GList of sessions. */ +static OwrSession * get_session_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id) +{ + GHashTableIter iter; + OwrSession *session; + gpointer session_id = GUINT_TO_POINTER(0); + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&session)) { + if (_owr_session_get_stream_id(session) == stream_id) { + AGENT_SESSIONS_UNLOCK(transport_agent); + g_object_ref(session); + return session; + } + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + g_warn_if_reached(); + return NULL; +} + +static GSList * get_sessions_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id) +{ + GSList *sessions = NULL; + GHashTableIter iter; + OwrSession *session; + gpointer session_id = GUINT_TO_POINTER(0); + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&session)) { + if (_owr_session_get_stream_id(session) == stream_id) { + sessions = g_slist_prepend(sessions, g_object_ref(session)); + } + } + sessions = g_slist_reverse(sessions); + AGENT_SESSIONS_UNLOCK(transport_agent); + + return sessions; +} + +static OwrPayload * get_payload(OwrTransportAgent *transport_agent, guint pt, OwrMediaSession **media_session) +{ + GHashTableIter iter; + OwrSession *s; + gpointer session_id = GUINT_TO_POINTER(0); + OwrPayload *payload = NULL; + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&s)) { + if (!OWR_IS_MEDIA_SESSION(s)) + continue; + + payload = _owr_media_session_get_receive_payload(OWR_MEDIA_SESSION(s), pt); + if (payload) { + if (media_session) + *media_session = g_object_ref(OWR_MEDIA_SESSION(s)); + AGENT_SESSIONS_UNLOCK(transport_agent); + return payload; + } + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + g_warn_if_reached(); + return NULL; } /* pad is transfer full */ @@ -2172,13 +2438,16 @@ static void on_caps(GstElement *sink, GParamSpec *pspec, OwrSession *session) g_object_get(sink, "caps", &caps, NULL); - if (GST_IS_CAPS(caps)) + if (GST_IS_CAPS(caps)) { GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Sending media configured with caps: %" GST_PTR_FORMAT, caps); + gst_caps_unref(caps); + } } static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMediaSession *media_session, OwrPayload * payload) { guint stream_id; + guint session_id; GstElement *send_input_bin = NULL; GstElement *encoder = NULL, *parser = NULL, *payloader = NULL, *rtp_capsfilter = NULL, *rtpbin = NULL; @@ -2187,36 +2456,39 @@ static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMedia gboolean link_ok = TRUE, sync_ok = TRUE; GstPad *sink_pad = NULL, *rtp_sink_pad = NULL, *rtp_capsfilter_src_pad = NULL, *ghost_src_pad = NULL, *encoder_sink_pad; + OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; OwrMediaType media_type; - GstPadLinkReturn link_res; guint send_ssrc = 0; gchar *cname = NULL; + OwrMediaSource *media_source = NULL; + GstElement *first = NULL; g_return_if_fail(transport_agent); g_return_if_fail(media_session); g_assert(payload); - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); + session_id = get_session_id_unlocked(transport_agent, OWR_SESSION(media_session)); - name = g_strdup_printf("send-input-bin-%u", stream_id); + name = g_strdup_printf("send-input-bin-%u-%u", session_id, stream_id); send_input_bin = gst_bin_new(name); - g_free(name); gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_input_bin); if (!gst_element_sync_state_with_parent(send_input_bin)) { - GST_ERROR("Failed to sync send-input-bin-%u state with parent", stream_id); + GST_ERROR("Failed to sync %s state with parent", name); + g_free(name); return; } + g_free(name); rtpbin = transport_agent->priv->rtpbin; - name = g_strdup_printf("send_rtp_sink_%u", stream_id); + name = g_strdup_printf("send_rtp_sink_%u", session_id); rtp_sink_pad = gst_element_get_request_pad(rtpbin, name); g_free(name); - link_rtpbin_to_send_output_bin(transport_agent, stream_id, TRUE, TRUE); + link_rtpbin_to_send_output_bin(transport_agent, session_id, stream_id, TRUE, TRUE); - g_object_get(payload, "media-type", &media_type, NULL); + g_object_get(payload, "media-type", &media_type, "codec-type", &codec_type, NULL); name = g_strdup_printf("send-rtp-capsfilter-%u", stream_id); rtp_capsfilter = gst_element_factory_make("capsfilter", name); @@ -2243,119 +2515,95 @@ static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMedia g_object_set(rtp_capsfilter, "caps", rtp_caps, NULL); gst_caps_unref(rtp_caps); - gst_bin_add(GST_BIN(send_input_bin), rtp_capsfilter); - - rtp_capsfilter_src_pad = gst_element_get_static_pad(rtp_capsfilter, "src"); - name = g_strdup_printf("src_%u", stream_id); - ghost_src_pad = ghost_pad_and_add_to_bin(rtp_capsfilter_src_pad, send_input_bin, name); - gst_object_unref(rtp_capsfilter_src_pad); - g_free(name); - link_res = gst_pad_link(ghost_src_pad, rtp_sink_pad); - g_warn_if_fail(link_res == GST_PAD_LINK_OK); - gst_object_unref(rtp_sink_pad); + media_source = _owr_media_session_get_send_source(media_session); - sync_ok &= gst_element_sync_state_with_parent(rtp_capsfilter); - g_warn_if_fail(sync_ok); - - if (media_type == OWR_MEDIA_TYPE_VIDEO) { - GstElement *gldownload, *flip, *queue = NULL, *encoder_capsfilter; - - name = g_strdup_printf("send-input-video-gldownload-%u", stream_id); - gldownload = gst_element_factory_make("gldownload", name); - g_free(name); - - name = g_strdup_printf("send-input-video-flip-%u", stream_id); - flip = gst_element_factory_make("videoflip", name); - g_assert(flip); - g_free(name); - g_return_if_fail(OWR_IS_VIDEO_PAYLOAD(payload)); - g_signal_connect_object(payload, "notify::rotation", G_CALLBACK(update_flip_method), flip, 0); - g_signal_connect_object(payload, "notify::mirror", G_CALLBACK(update_flip_method), flip, 0); - update_flip_method(payload, NULL, flip); - - name = g_strdup_printf("send-input-video-queue-%u", stream_id); - queue = gst_element_factory_make("queue", name); - g_free(name); - g_object_set(queue, "max-size-buffers", 3, "max-size-bytes", 0, - "max-size-time", G_GUINT64_CONSTANT(0), NULL); + if (media_type == OWR_MEDIA_TYPE_VIDEO && OWR_IS_VIDEO_PAYLOAD(payload)) { + GstElement *gldownload; + GstElement *flip = NULL, *queue = NULL, *encoder_capsfilter = NULL; + if (_owr_codec_type_is_raw(_owr_payload_get_codec_type(payload))) { + name = g_strdup_printf("send-input-video-gldownload-%u", stream_id); + gldownload = gst_element_factory_make("gldownload", name); + g_free(name); + gst_bin_add(GST_BIN(send_input_bin), gldownload); + } + if (!_owr_media_source_supports_interfaces(media_source, OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION)) { + name = g_strdup_printf("send-input-video-flip-%u", stream_id); + flip = gst_element_factory_make("videoflip", name); + g_assert(flip); + g_free(name); + g_signal_connect_object(payload, "notify::rotation", G_CALLBACK(_owr_update_flip_method), flip, 0); + g_signal_connect_object(payload, "notify::mirror", G_CALLBACK(_owr_update_flip_method), flip, 0); + _owr_update_flip_method(G_OBJECT(payload), NULL, flip); + + name = g_strdup_printf("send-input-video-queue-%u", stream_id); + queue = gst_element_factory_make("queue", name); + g_free(name); + g_object_set(queue, "max-size-buffers", 3, "max-size-bytes", 0, + "max-size-time", G_GUINT64_CONSTANT(0), NULL); + + encoder = _owr_payload_create_encoder(payload); + parser = _owr_create_parser(_owr_payload_get_codec_type(payload)); + + g_warn_if_fail(encoder); + + encoder_sink_pad = gst_element_get_static_pad(encoder, "sink"); + g_signal_connect(encoder_sink_pad, "notify::caps", G_CALLBACK(on_caps), OWR_SESSION(media_session)); + gst_object_unref(encoder_sink_pad); + + name = g_strdup_printf("send-input-video-encoder-capsfilter-%u", stream_id); + encoder_capsfilter = gst_element_factory_make("capsfilter", name); + g_free(name); + caps = _owr_payload_create_encoded_caps(payload); + g_object_set(encoder_capsfilter, "caps", caps, NULL); + gst_caps_unref(caps); + + gst_bin_add_many(GST_BIN(send_input_bin), flip, queue, encoder, NULL); + + if (parser) + gst_bin_add(GST_BIN(send_input_bin), parser); + gst_bin_add(GST_BIN(send_input_bin), encoder_capsfilter); + } + } else { /* Audio */ encoder = _owr_payload_create_encoder(payload); - parser = _owr_payload_create_parser(payload); - payloader = _owr_payload_create_payload_packetizer(payload); - g_warn_if_fail(payloader && encoder); + parser = _owr_create_parser(_owr_payload_get_codec_type(payload)); encoder_sink_pad = gst_element_get_static_pad(encoder, "sink"); g_signal_connect(encoder_sink_pad, "notify::caps", G_CALLBACK(on_caps), OWR_SESSION(media_session)); gst_object_unref(encoder_sink_pad); - name = g_strdup_printf("send-input-video-encoder-capsfilter-%u", stream_id); - encoder_capsfilter = gst_element_factory_make("capsfilter", name); - g_free(name); - caps = _owr_payload_create_encoded_caps(payload); - g_object_set(encoder_capsfilter, "caps", caps, NULL); - gst_caps_unref(caps); - - gst_bin_add_many(GST_BIN(send_input_bin), gldownload, flip, queue, encoder, encoder_capsfilter, payloader, NULL); - if (parser) { + gst_bin_add(GST_BIN(send_input_bin), encoder); + if (parser) gst_bin_add(GST_BIN(send_input_bin), parser); - link_ok &= gst_element_link_many(gldownload, flip, queue, encoder, parser, encoder_capsfilter, payloader, NULL); - } else - link_ok &= gst_element_link_many(gldownload, flip, queue, encoder, encoder_capsfilter, payloader, NULL); - - link_ok &= gst_element_link_many(payloader, rtp_capsfilter, NULL); + } - g_warn_if_fail(link_ok); + payloader = _owr_payload_create_payload_packetizer(payload); + g_assert(payloader); + gst_bin_add_many(GST_BIN(send_input_bin), payloader, rtp_capsfilter, NULL); - sync_ok &= gst_element_sync_state_with_parent(rtp_capsfilter); - sync_ok &= gst_element_sync_state_with_parent(payloader); - if (parser) - sync_ok &= gst_element_sync_state_with_parent(parser); - sync_ok &= gst_element_sync_state_with_parent(encoder_capsfilter); - sync_ok &= gst_element_sync_state_with_parent(encoder); - sync_ok &= gst_element_sync_state_with_parent(queue); - sync_ok &= gst_element_sync_state_with_parent(flip); - sync_ok &= gst_element_sync_state_with_parent(gldownload); - - name = g_strdup_printf("video_sink_%u_%u", OWR_CODEC_TYPE_NONE, stream_id); - sink_pad = gst_element_get_static_pad(gldownload, "sink"); - add_pads_to_bin_and_transport_bin(sink_pad, send_input_bin, - transport_agent->priv->transport_bin, name); - gst_object_unref(sink_pad); - g_free(name); - } else { /* Audio */ - encoder = _owr_payload_create_encoder(payload); - parser = _owr_payload_create_parser(payload); - payloader = _owr_payload_create_payload_packetizer(payload); + _owr_bin_link_and_sync_elements(GST_BIN(send_input_bin), &link_ok, &sync_ok, &first, NULL); + g_warn_if_fail(link_ok && sync_ok); - encoder_sink_pad = gst_element_get_static_pad(encoder, "sink"); - g_signal_connect(encoder_sink_pad, "notify::caps", G_CALLBACK(on_caps), OWR_SESSION(media_session)); - gst_object_unref(encoder_sink_pad); + name = g_strdup_printf("%s_sink_%u_%u", media_type == OWR_MEDIA_TYPE_VIDEO ? "video" : "audio", + codec_type, stream_id); + sink_pad = gst_element_get_static_pad(first, "sink"); + add_pads_to_bin_and_transport_bin(sink_pad, send_input_bin, + transport_agent->priv->transport_bin, name); + gst_object_unref(sink_pad); + g_free(name); - gst_bin_add_many(GST_BIN(send_input_bin), encoder, payloader, NULL); - if (parser) { - gst_bin_add(GST_BIN(send_input_bin), parser); - link_ok &= gst_element_link_many(encoder, parser, payloader, NULL); - } else - link_ok &= gst_element_link_many(encoder, payloader, NULL); + rtp_capsfilter_src_pad = gst_element_get_static_pad(rtp_capsfilter, "src"); + name = g_strdup_printf("src_%u", stream_id); + ghost_src_pad = ghost_pad_and_add_to_bin(rtp_capsfilter_src_pad, send_input_bin, name); + gst_object_unref(rtp_capsfilter_src_pad); + g_free(name); + g_warn_if_fail(gst_pad_link(ghost_src_pad, rtp_sink_pad) == GST_PAD_LINK_OK); + gst_object_unref(rtp_sink_pad); - link_ok &= gst_element_link_many(payloader, rtp_capsfilter, NULL); - g_warn_if_fail(link_ok); + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(send_input_bin), GST_DEBUG_GRAPH_SHOW_ALL, "send-input-bin"); - sync_ok &= gst_element_sync_state_with_parent(rtp_capsfilter); - sync_ok &= gst_element_sync_state_with_parent(payloader); - if (parser) - sync_ok &= gst_element_sync_state_with_parent(parser); - sync_ok &= gst_element_sync_state_with_parent(encoder); - g_warn_if_fail(sync_ok); - - name = g_strdup_printf("audio_raw_sink_%u", stream_id); - sink_pad = gst_element_get_static_pad(encoder, "sink"); - add_pads_to_bin_and_transport_bin(sink_pad, send_input_bin, - transport_agent->priv->transport_bin, name); - gst_object_unref(sink_pad); - g_free(name); - } + g_object_unref(media_source); } static void on_new_remote_candidate(OwrTransportAgent *transport_agent, gboolean forced, OwrSession *session) @@ -2370,8 +2618,7 @@ static void on_new_remote_candidate(OwrTransportAgent *transport_agent, gboolean g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); g_return_if_fail(OWR_IS_SESSION(session)); - stream_id = get_stream_id(transport_agent, session); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(session); item = forced ? _owr_session_get_forced_remote_candidates(session) : _owr_session_get_remote_candidates(session); @@ -2428,8 +2675,7 @@ static void on_local_candidate_change(OwrTransportAgent *transport_agent, OwrCan g_return_if_fail(OWR_IS_CANDIDATE(candidate)); g_return_if_fail(OWR_IS_SESSION(session)); - stream_id = get_stream_id(transport_agent, session); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(session); g_object_get(G_OBJECT(candidate), "ufrag", &ufrag, "password", &password, NULL); nice_agent_set_local_credentials(transport_agent->priv->nice_agent, stream_id, ufrag, password); @@ -2454,7 +2700,7 @@ static gboolean emit_on_incoming_source(GHashTable *args) } static void signal_incoming_source(OwrMediaType type, OwrTransportAgent *transport_agent, - guint stream_id, OwrCodecType codec_type) + guint session_id, guint stream_id, OwrCodecType codec_type) { OwrMediaSession *media_session; OwrMediaSource *source; @@ -2462,9 +2708,9 @@ static void signal_incoming_source(OwrMediaType type, OwrTransportAgent *transpo g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, stream_id)); + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - source = _owr_remote_media_source_new(type, stream_id, codec_type, + source = _owr_remote_media_source_new(type, session_id, stream_id, codec_type, transport_agent->priv->transport_bin); g_return_if_fail(source); @@ -2483,6 +2729,7 @@ static void on_transport_bin_pad_added(GstElement *transport_bin, GstPad *new_pa OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; guint stream_id = 0; + guint session_id = 0; g_return_if_fail(GST_IS_BIN(transport_bin)); g_return_if_fail(GST_IS_PAD(new_pad)); @@ -2493,14 +2740,14 @@ static void on_transport_bin_pad_added(GstElement *transport_bin, GstPad *new_pa if (g_str_has_prefix(new_pad_name, "audio_raw_src")) { media_type = OWR_MEDIA_TYPE_AUDIO; codec_type = OWR_CODEC_TYPE_NONE; - sscanf(new_pad_name, "audio_raw_src_%u", &stream_id); + sscanf(new_pad_name, "audio_raw_src_%u_%u", &session_id, &stream_id); } else if (g_str_has_prefix(new_pad_name, "video_src_")) { media_type = OWR_MEDIA_TYPE_VIDEO; - sscanf(new_pad_name, "video_src_%u_%u", &codec_type, &stream_id); + sscanf(new_pad_name, "video_src_%u_%u_%u", &codec_type, &session_id, &stream_id); } - if (media_type != OWR_MEDIA_TYPE_UNKNOWN && codec_type == OWR_CODEC_TYPE_NONE) - signal_incoming_source(media_type, transport_agent, stream_id, codec_type); + if (media_type != OWR_MEDIA_TYPE_UNKNOWN) + signal_incoming_source(media_type, transport_agent, session_id, stream_id, codec_type); g_free(new_pad_name); } @@ -2515,46 +2762,55 @@ static void on_rtpbin_pad_added(GstElement *rtpbin, GstPad *new_pad, OwrTranspor new_pad_name = gst_pad_get_name(new_pad); if (g_str_has_prefix(new_pad_name, "recv_rtp_src_")) { - guint32 session_id = 0, ssrc = 0, pt = 0; + guint32 stream_id = 0, ssrc = 0, pt = 0, session_id = 0; OwrMediaSession *media_session = NULL; OwrPayload *payload = NULL; OwrMediaType media_type; sscanf(new_pad_name, "recv_rtp_src_%u_%u_%u", &session_id, &ssrc, &pt); + g_free(new_pad_name); media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); + + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); payload = _owr_media_session_get_receive_payload(media_session, pt); - g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); g_return_if_fail(OWR_IS_PAYLOAD(payload)); + g_object_get(payload, "media-type", &media_type, NULL); g_object_set_data(G_OBJECT(media_session), "ssrc", GUINT_TO_POINTER(ssrc)); + session_id = get_session_id(transport_agent, OWR_SESSION(media_session)); + if (media_type == OWR_MEDIA_TYPE_VIDEO) - setup_video_receive_elements(new_pad, session_id, payload, transport_agent); + setup_video_receive_elements(new_pad, session_id, stream_id, payload, transport_agent); else - setup_audio_receive_elements(new_pad, session_id, payload, transport_agent); + setup_audio_receive_elements(new_pad, session_id, stream_id, payload, transport_agent); /* Hook up RTCP sending if it isn't already */ - link_rtpbin_to_send_output_bin(transport_agent, session_id, FALSE, TRUE); + link_rtpbin_to_send_output_bin(transport_agent, session_id, stream_id, FALSE, TRUE); - g_object_unref(media_session); + if (media_session) + g_object_unref(media_session); g_object_unref(payload); } else if (g_str_has_prefix(new_pad_name, "send_rtp_src")) { - guint32 session_id = 0; - sscanf(new_pad_name, "send_rtp_src_%u", &session_id); + guint32 stream_id = 0; + sscanf(new_pad_name, "send_rtp_src_%u", &stream_id); + g_free(new_pad_name); } - g_free(new_pad_name); } typedef struct { OwrSession *session; - guint session_id; + guint stream_id; } SessionData; -static void session_data_free(gpointer session_data) +static void session_data_free(gpointer user_data) { + SessionData *session_data = (SessionData *)user_data; + g_object_unref(session_data->session); g_slice_free(SessionData, session_data); } @@ -2566,82 +2822,90 @@ static GstPadProbeReturn check_for_keyframe(GstPad *pad, GstPadProbeInfo *info, if (!GST_BUFFER_FLAG_IS_SET(info->data, GST_BUFFER_FLAG_DELTA_UNIT)) { GST_CAT_INFO_OBJECT(_owrsession_debug, session_data->session, - "Session %u, Received keyframe for %u\n", session_data->session_id, + "Session %u, Received keyframe for %u", session_data->stream_id, GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(session_data->session), "ssrc"))); } return GST_PAD_PROBE_OK; } -static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent) +static gboolean force_key_unit_event(gpointer data) +{ + GstElement *videorepair = GST_ELEMENT_CAST(data); + GstPad *sink_pad = gst_element_get_static_pad(videorepair, "sink"); + gst_pad_push_event(sink_pad, gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE, FALSE, 0)); + gst_object_unref(sink_pad); + gst_object_unref(videorepair); + return G_SOURCE_REMOVE; +} + +static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent) { - GstPad *depay_sink_pad = NULL, *ghost_pad = NULL; + GstPad *sink_pad = NULL, *ghost_pad = NULL; gboolean sync_ok = TRUE; GstElement *receive_output_bin; - GstElement *rtpdepay, *videorepair1, *parser, *decoder; + GstElement *rtpdepay, *videorepair1, *parser; GstPadLinkReturn link_res; gboolean link_ok = TRUE; OwrCodecType codec_type; gchar name[100]; GstPad *pad; SessionData *session_data; + GstElement *first = NULL, *last = NULL; - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "receive-output-bin-%u", session_id); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "receive-output-bin-%u-%u", session_id, stream_id); receive_output_bin = gst_bin_new(name); gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), receive_output_bin); if (!gst_element_sync_state_with_parent(receive_output_bin)) { - GST_ERROR("Failed to sync receive-output-bin-%u state with parent", session_id); + GST_ERROR("Failed to sync %s state with parent", name); return; } rtpdepay = _owr_payload_create_payload_depacketizer(payload); - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "videorepair1_%u", session_id); + gst_bin_add(GST_BIN(receive_output_bin), rtpdepay); + + codec_type = _owr_payload_get_codec_type(payload); + parser = _owr_create_parser(codec_type); + if (parser) + gst_bin_add(GST_BIN(receive_output_bin), parser); + + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "videorepair1_%u", stream_id); videorepair1 = gst_element_factory_make("videorepair", name); + gst_bin_add(GST_BIN(receive_output_bin), videorepair1); pad = gst_element_get_static_pad(videorepair1, "src"); session_data = g_slice_new(SessionData); session_data->session = get_session(transport_agent, session_id); - session_data->session_id = session_id; + session_data->stream_id = stream_id; gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_BUFFER, check_for_keyframe, session_data, session_data_free); gst_object_unref(pad); pad = NULL; - g_object_get(payload, "codec-type", &codec_type, NULL); - parser = _owr_payload_create_parser(payload); - decoder = _owr_payload_create_decoder(payload); + // The OMX video decoder and vp8dec element don't seem to handle very well the incoming + // streams from Chrome, not taking into account the first intra frame + // received. So for now, force a PLI request towards the sender after 250ms. + g_timeout_add(250, force_key_unit_event, gst_object_ref(videorepair1)); - gst_bin_add_many(GST_BIN(receive_output_bin), rtpdepay, - videorepair1, decoder, /*decoded_tee,*/ NULL); - depay_sink_pad = gst_element_get_static_pad(rtpdepay, "sink"); - if (parser) { - gst_bin_add(GST_BIN(receive_output_bin), parser); - link_ok &= gst_element_link_many(rtpdepay, parser, videorepair1, decoder, NULL); - } else - link_ok &= gst_element_link_many(rtpdepay, videorepair1, decoder, NULL); + _owr_bin_link_and_sync_elements(GST_BIN(receive_output_bin), &link_ok, &sync_ok, &first, &last); + g_warn_if_fail(link_ok && sync_ok); - ghost_pad = ghost_pad_and_add_to_bin(depay_sink_pad, receive_output_bin, "sink"); + sink_pad = gst_element_get_static_pad(first, "sink"); + ghost_pad = ghost_pad_and_add_to_bin(sink_pad, receive_output_bin, "sink"); link_res = gst_pad_link(new_pad, ghost_pad); - gst_object_unref(depay_sink_pad); + gst_object_unref(sink_pad); ghost_pad = NULL; - g_warn_if_fail(link_ok && (link_res == GST_PAD_LINK_OK)); - - sync_ok &= gst_element_sync_state_with_parent(decoder); - if (parser) - sync_ok &= gst_element_sync_state_with_parent(parser); - sync_ok &= gst_element_sync_state_with_parent(videorepair1); - sync_ok &= gst_element_sync_state_with_parent(rtpdepay); - g_warn_if_fail(sync_ok); + g_warn_if_fail(link_res == GST_PAD_LINK_OK); - pad = gst_element_get_static_pad(decoder, "src"); - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_src_%u_%u", OWR_CODEC_TYPE_NONE, - session_id); + pad = gst_element_get_static_pad(last, "src"); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_src_%u_%u_%u", codec_type, + session_id, stream_id); add_pads_to_bin_and_transport_bin(pad, receive_output_bin, transport_agent->priv->transport_bin, name); gst_object_unref(pad); } -static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent) +static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent) { GstElement *receive_output_bin; gchar *pad_name = NULL; @@ -2652,7 +2916,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow gboolean link_ok = FALSE; gboolean sync_ok = TRUE; - pad_name = g_strdup_printf("receive-output-bin-%u", session_id); + pad_name = g_strdup_printf("receive-output-bin-%u-%u", session_id, stream_id); receive_output_bin = gst_bin_new(pad_name); g_free(pad_name); pad_name = NULL; @@ -2660,7 +2924,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), receive_output_bin); sync_ok &= gst_element_sync_state_with_parent(receive_output_bin); - element_name = g_strdup_printf("recv-rtp-capsfilter-%u", session_id); + element_name = g_strdup_printf("recv-rtp-capsfilter-%u", stream_id); rtp_capsfilter = gst_element_factory_make("capsfilter", element_name); g_free(element_name); rtp_caps = _owr_payload_create_rtp_caps(payload); @@ -2669,8 +2933,8 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow rtpdepay = _owr_payload_create_payload_depacketizer(payload); - parser = _owr_payload_create_parser(payload); - decoder = _owr_payload_create_decoder(payload); + parser = _owr_create_parser(_owr_payload_get_codec_type(payload)); + decoder = _owr_create_decoder(_owr_payload_get_codec_type(payload)); gst_bin_add_many(GST_BIN(receive_output_bin), rtp_capsfilter, rtpdepay, decoder, NULL); @@ -2687,7 +2951,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow ghost_pad = ghost_pad_and_add_to_bin(rtp_caps_sink_pad, receive_output_bin, "sink"); gst_object_unref(rtp_caps_sink_pad); if (!GST_PAD_LINK_SUCCESSFUL(gst_pad_link(new_pad, ghost_pad))) { - GST_ERROR("Failed to link rtpbin with receive-output-bin-%u", session_id); + GST_ERROR("Failed to link rtpbin with receive-output-bin-%u", stream_id); return; } ghost_pad = NULL; @@ -2700,7 +2964,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow g_warn_if_fail(sync_ok); pad = gst_element_get_static_pad(decoder, "src"); - pad_name = g_strdup_printf("audio_raw_src_%u", session_id); + pad_name = g_strdup_printf("audio_raw_src_%u_%u", session_id, stream_id); add_pads_to_bin_and_transport_bin(pad, receive_output_bin, transport_agent->priv->transport_bin, pad_name); gst_object_unref(pad); @@ -2717,16 +2981,22 @@ static GstCaps * on_rtpbin_request_pt_map(GstElement *rtpbin, guint session_id, g_return_val_if_fail(rtpbin, NULL); g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), NULL); - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - g_return_val_if_fail(OWR_IS_MEDIA_SESSION(media_session), NULL); + if (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + payload = get_payload(transport_agent, pt, NULL); + } else { + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + g_return_val_if_fail(OWR_IS_MEDIA_SESSION(media_session), NULL); - payload = _owr_media_session_get_receive_payload(media_session, pt); + payload = _owr_media_session_get_receive_payload(media_session, pt); + } if (payload) { caps = _owr_payload_create_rtp_caps(payload); g_object_unref(payload); } - g_object_unref(media_session); + + if (media_session) + g_object_unref(media_session); return caps; } @@ -2739,6 +3009,7 @@ static GstElement * create_aux_bin(gchar *prefix, GstElement *rtx, guint session tmp = g_strdup_printf("%s_%u", prefix, session_id); bin = gst_bin_new(tmp); + GST_DEBUG("Retransmission auxiliary bin created: %s", tmp); g_free(tmp); gst_bin_add(GST_BIN(bin), rtx); @@ -2772,7 +3043,7 @@ static GstElement * on_rtpbin_request_aux_sender(G_GNUC_UNUSED GstElement *rtpbi guint pt, rtx_time; gchar *tmp; - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + media_session = OWR_MEDIA_SESSION(get_session_unlocked(transport_agent, session_id)); g_return_val_if_fail(media_session, NULL); payload = _owr_media_session_get_send_payload(media_session); @@ -2808,21 +3079,36 @@ static GstElement * on_rtpbin_request_aux_sender(G_GNUC_UNUSED GstElement *rtpbi return create_aux_bin("rtprtxsend", rtxsend, session_id); no_retransmission: + GST_DEBUG("Retransmission support disabled on sending side"); return NULL; } -static GstElement * on_rtpbin_request_aux_receiver(G_GNUC_UNUSED GstElement *rtpbin, G_GNUC_UNUSED guint session_id, OwrTransportAgent *transport_agent) +static GstElement * on_rtpbin_request_aux_receiver(G_GNUC_UNUSED GstElement *rtpbin, guint session_id, OwrTransportAgent *transport_agent) { OwrMediaSession *media_session; - GstElement *rtxrecv; + GstElement *rtxrecv = NULL; GstStructure *pt_map; - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + media_session = OWR_MEDIA_SESSION(get_session_unlocked(transport_agent, session_id)); g_return_val_if_fail(media_session, NULL); pt_map = _owr_media_session_get_receive_rtx_pt_map(media_session); g_object_unref(media_session); + if ((transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) && !pt_map) { + GSList* walk; + GST_DEBUG("no valid pt_map found, looking for one in the staged sessions"); + for (walk = transport_agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { + OwrSession* session = OWR_SESSION(walk->data); + if (!OWR_IS_MEDIA_SESSION(session)) + continue; + pt_map = _owr_media_session_get_receive_rtx_pt_map(OWR_MEDIA_SESSION(session)); + GST_DEBUG("pt_map found on staged session: %p", pt_map); + if (pt_map) + break; + } + } + if (!pt_map) goto no_retransmission; @@ -2836,13 +3122,14 @@ static GstElement * on_rtpbin_request_aux_receiver(G_GNUC_UNUSED GstElement *rtp return create_aux_bin("rtprtxrecv", rtxrecv, session_id); no_retransmission: + GST_DEBUG("Retransmission support disabled on receiving side"); return NULL; } -static void print_rtcp_type(GObject *session, guint session_id, +static void print_rtcp_type(GObject *session, guint stream_id, GstRTCPType packet_type) { - GST_CAT_DEBUG_OBJECT(_owrsession_debug, session, "Session %u, Received RTCP %s\n", session_id, + GST_CAT_DEBUG_OBJECT(_owrsession_debug, session, "Session %u, Received RTCP %s", stream_id, packet_type == GST_RTCP_TYPE_INVALID ? "Invalid type (INVALID)" : packet_type == GST_RTCP_TYPE_SR ? "Sender Report (SR)" : packet_type == GST_RTCP_TYPE_RR ? "Receiver Report (RR)" : @@ -2854,77 +3141,77 @@ static void print_rtcp_type(GObject *session, guint session_id, "unknown"); } -static void print_rtcp_feedback_type(GObject *session, guint session_id, +static void print_rtcp_feedback_type(GObject *session, guint stream_id, guint fbtype, guint media_ssrc, GstRTCPType packet_type, guint8 *fci, gboolean is_received) { if (fbtype == GST_RTCP_FB_TYPE_INVALID) { - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Invalid type\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Invalid type", + stream_id, is_received ? "Received" : "Sent", media_ssrc); } else if (packet_type == GST_RTCP_TYPE_RTPFB) { switch (fbtype) { case GST_RTCP_RTPFB_TYPE_NACK: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Generic NACK\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Generic NACK", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_TMMBR: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Request\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Request", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_TMMBN: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Notification\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Notification", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_RTCP_SR_REQ: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Request an SR packet for early synchronization\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Request an SR packet for early synchronization", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_SCREAM: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: SCReAM\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: SCReAM", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; default: - GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u\n", - session_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); + GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u", + stream_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); break; } } else if (packet_type == GST_RTCP_TYPE_PSFB) { switch (fbtype) { case GST_RTCP_PSFB_TYPE_PLI: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Picture Loss Indication\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Picture Loss Indication", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_SLI: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Slice Loss Indication\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Slice Loss Indication", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_RPSI: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Reference Picture Selection Indication\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Reference Picture Selection Indication", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_AFB: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Application layer Feedback\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Application layer Feedback", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_FIR: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Full Intra Request Command\n", - session_id, is_received ? "Received" : "Sent", fci ? GST_READ_UINT32_BE(fci) : 0); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Full Intra Request Command", + stream_id, is_received ? "Received" : "Sent", fci ? GST_READ_UINT32_BE(fci) : 0); break; case GST_RTCP_PSFB_TYPE_TSTR: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Request\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Request", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_TSTN: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Notification\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Notification", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_VBCN: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Video Back Channel Message\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Video Back Channel Message", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; default: - GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u\n", - session_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); + GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u", + stream_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); break; } } @@ -2939,17 +3226,15 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea GstRTCPType packet_type; gboolean has_packet, do_not_suppress = FALSE; OwrMediaSession *media_session; - OwrPayload *send_payload; - OwrMediaType media_type = -1; GValueArray *sources = NULL; GObject *source = NULL; - guint session_id = 0, rtcp_session_id = 0; + guint stream_id = 0, rtcp_stream_id = 0; GList *it, *next; GHashTable *rtcp_info; OWR_UNUSED(early); - session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session_id")); + stream_id = GPOINTER_TO_UINT(g_object_get_data(session, "stream-id")); if (gst_rtcp_buffer_map(buffer, GST_MAP_READ | GST_MAP_WRITE, &rtcp_buffer)) { guint pt, fmt, ssrc, last_fb_wc, highest_seq, n_loss, n_ecn; @@ -2957,9 +3242,9 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea has_packet = gst_rtcp_buffer_get_first_packet(&rtcp_buffer, &rtcp_packet); for (; has_packet; has_packet = gst_rtcp_packet_move_to_next(&rtcp_packet)) { packet_type = gst_rtcp_packet_get_type(&rtcp_packet); - print_rtcp_type(session, session_id, packet_type); + print_rtcp_type(session, stream_id, packet_type); if (packet_type == GST_RTCP_TYPE_PSFB || packet_type == GST_RTCP_TYPE_RTPFB) { - print_rtcp_feedback_type(session, session_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), + print_rtcp_feedback_type(session, stream_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), gst_rtcp_packet_fb_get_media_ssrc(&rtcp_packet), packet_type, gst_rtcp_packet_fb_get_fci(&rtcp_packet), FALSE); do_not_suppress = TRUE; @@ -2974,10 +3259,10 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea pt = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "pt")); ssrc = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "ssrc")); - rtcp_session_id = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "session-id")); + rtcp_stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "stream_id")); if (pt == GST_RTCP_TYPE_RTPFB) { - if (session_id != rtcp_session_id) { + if (stream_id != rtcp_stream_id) { it = g_list_next(it); continue; } @@ -3032,19 +3317,13 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(agent), do_not_suppress); - media_session = OWR_MEDIA_SESSION(get_session(agent, session_id)); + media_session = OWR_MEDIA_SESSION(g_object_get_data(session, "session")); g_return_val_if_fail(OWR_IS_MEDIA_SESSION(media_session), do_not_suppress); - send_payload = _owr_media_session_get_send_payload(media_session); - if (send_payload) { - g_object_get(send_payload, "media-type", &media_type, NULL); - g_object_unref(send_payload); - } g_object_get(session, "sources", &sources, NULL); source = g_value_get_object(g_value_array_get_nth(sources, 0)); prepare_rtcp_stats(media_session, source); g_value_array_free(sources); - g_object_unref(media_session); return do_not_suppress; } @@ -3056,19 +3335,19 @@ static void on_receiving_rtcp(GObject *session, GstBuffer *buffer, GstRTCPPacket rtcp_packet; GstRTCPType packet_type; gboolean has_packet; - guint session_id = 0; + guint stream_id = 0; OWR_UNUSED(agent); - session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session_id")); + stream_id = GPOINTER_TO_UINT(g_object_get_data(session, "stream-id")); if (gst_rtcp_buffer_map(buffer, GST_MAP_READ, &rtcp_buffer)) { has_packet = gst_rtcp_buffer_get_first_packet(&rtcp_buffer, &rtcp_packet); for (; has_packet; has_packet = gst_rtcp_packet_move_to_next(&rtcp_packet)) { packet_type = gst_rtcp_packet_get_type(&rtcp_packet); - print_rtcp_type(session, session_id, packet_type); + print_rtcp_type(session, stream_id, packet_type); if (packet_type == GST_RTCP_TYPE_PSFB || packet_type == GST_RTCP_TYPE_RTPFB) { - print_rtcp_feedback_type(session, session_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), + print_rtcp_feedback_type(session, stream_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), gst_rtcp_packet_fb_get_media_ssrc(&rtcp_packet), packet_type, gst_rtcp_packet_fb_get_fci(&rtcp_packet), TRUE); break; @@ -3132,6 +3411,7 @@ static void on_ssrc_active(GstElement *rtpbin, guint session_id, guint ssrc, GObject *rtp_session, *rtp_source; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); @@ -3143,13 +3423,38 @@ static void on_ssrc_active(GstElement *rtpbin, guint session_id, guint ssrc, g_object_unref(media_session); } +static guint on_bundled_ssrc(GstElement *rtpbin, guint ssrc, OwrTransportAgent *transport_agent) +{ + GHashTableIter iter; + OwrSession *session; + gpointer session_id = GUINT_TO_POINTER(0); + guint found_session_id = 0; + + OWR_UNUSED(rtpbin); + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&session)) { + guint receive_ssrc, receive_rtx_ssrc; + if (!OWR_IS_MEDIA_SESSION(session)) + continue; + g_object_get(OWR_MEDIA_SESSION(session), "receive-ssrc", &receive_ssrc, "receive-rtx-ssrc", &receive_rtx_ssrc, NULL); + if ((receive_rtx_ssrc == ssrc) || (receive_ssrc == ssrc)) { + found_session_id = GPOINTER_TO_UINT(session_id); + break; + } + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + return found_session_id; +} + static void on_new_jitterbuffer(G_GNUC_UNUSED GstElement *rtpbin, GstElement *jitterbuffer, guint session_id, G_GNUC_UNUSED guint ssrc, OwrTransportAgent *transport_agent) { OwrMediaSession *media_session; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); if (_owr_media_session_want_receive_rtx(media_session)) g_object_set(jitterbuffer, "do-retransmission", TRUE, NULL); @@ -3186,6 +3491,7 @@ static gboolean create_datachannel(OwrTransportAgent *transport_agent, guint32 s gboolean ordered, negotiated; gint max_packet_life_time, max_packet_retransmits, sctp_stream_id; gchar *protocol, *label; + OwrSession *session; g_object_get(data_channel, "ordered", &ordered, "max-packet-life-time", &max_packet_life_time, "max-retransmits", &max_packet_retransmits, "protocol", &protocol, "negotiated", &negotiated, @@ -3198,7 +3504,7 @@ static gboolean create_datachannel(OwrTransportAgent *transport_agent, guint32 s goto end; } - if (!negotiated && !is_valid_sctp_stream_id(transport_agent, session_id, sctp_stream_id, FALSE)) { + if (!negotiated && !is_valid_sctp_session_id(transport_agent, session_id, sctp_stream_id, FALSE)) { g_warning("Invalid stream_id"); g_free(protocol); g_free(label); @@ -3213,11 +3519,14 @@ static gboolean create_datachannel(OwrTransportAgent *transport_agent, guint32 s goto end; } + session = get_session(transport_agent, session_id); + data_channel_info = g_new0(DataChannel, 1); data_channel_info->state = OWR_DATA_CHANNEL_STATE_CONNECTING; data_channel_info->id = sctp_stream_id; data_channel_info->label = label; data_channel_info->protocol = protocol; + data_channel_info->stream_id = _owr_session_get_stream_id(session); data_channel_info->session_id = session_id; data_channel_info->ctrl_bytes_sent = 0; data_channel_info->negotiated = negotiated; @@ -3268,16 +3577,18 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, GstElement *data_sink, *receive_input_bin; GstPad *appsink_sinkpad; GstPadLinkReturn link_ret; - guint sctp_stream_id, session_id; + guint sctp_stream_id, stream_id; gchar *name; GstAppSinkCallbacks callbacks; DataChannel *data_channel_info; gboolean remotely_initiated = FALSE, valid_id; + OwrSession *session; + guint session_id; name = gst_pad_get_name(sctpdec_srcpad); sscanf(name, "src_%u", &sctp_stream_id); g_free(name); - session_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpdec), "session-id")); + stream_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpdec), "stream_id")); g_rw_lock_writer_lock(&priv->data_channels_rw_mutex); data_channel_info = (DataChannel *)g_hash_table_lookup(priv->data_channels, @@ -3286,7 +3597,7 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, remotely_initiated = TRUE; data_channel_info = g_new0(DataChannel, 1); data_channel_info->state = OWR_DATA_CHANNEL_STATE_CONNECTING; - data_channel_info->session_id = session_id; + data_channel_info->stream_id = stream_id; data_channel_info->label = NULL; data_channel_info->protocol = NULL; data_channel_info->negotiated = FALSE; @@ -3296,7 +3607,10 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, } g_rw_lock_writer_unlock(&priv->data_channels_rw_mutex); - valid_id = is_valid_sctp_stream_id(transport_agent, session_id, sctp_stream_id, + session = get_session_from_stream_id(transport_agent, stream_id); + session_id = get_session_id(transport_agent, session); + + valid_id = is_valid_sctp_session_id(transport_agent, session_id, sctp_stream_id, remotely_initiated); if (!data_channel_info->negotiated && !valid_id) { g_warning("Invalid stream_id"); @@ -3325,7 +3639,7 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, data_channel_info->data_sink = data_sink; data_channel_info->id = sctp_stream_id; - name = g_strdup_printf("receive-input-bin-%u", session_id); + name = g_strdup_printf("receive-input-bin-%u", stream_id); receive_input_bin = gst_bin_get_by_name(GST_BIN(priv->transport_bin), name); g_free(name); @@ -3368,8 +3682,8 @@ static void sctpdec_pad_removed(GstElement *sctpdec, GstPad *sctpdec_srcpad, already_closing = TRUE; else { data_channel_info->state = OWR_DATA_CHANNEL_STATE_CLOSING; - data_session = OWR_DATA_SESSION( - get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_assert(OWR_IS_DATA_SESSION(data_session)); data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); @@ -3421,15 +3735,15 @@ static void sctpdec_pad_removed(GstElement *sctpdec, GstPad *sctpdec_srcpad, static void on_sctp_association_established(GstElement *sctpenc, gboolean established, OwrTransportAgent *transport_agent) { - guint session_id; + guint stream_id; OwrDataSession *data_session; GList *data_channels, *it; transport_agent->priv->data_session_established = established; if (established) { g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "An SCTP association has been established"); - session_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpenc), "session-id")); - data_session = OWR_DATA_SESSION(get_session(transport_agent, session_id)); + stream_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpenc), "stream_id")); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, stream_id)); g_assert(data_session); data_channels = _owr_data_session_get_datachannels(data_session); @@ -3537,10 +3851,11 @@ static gboolean create_datachannel_appsrc(OwrTransportAgent *transport_agent, g_rw_lock_reader_unlock(&priv->data_channels_rw_mutex); g_assert(data_channel_info); - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_return_val_if_fail(data_session, FALSE); - name = g_strdup_printf("send-output-bin-%u", data_channel_info->session_id); + name = g_strdup_printf("send-output-bin-%u", data_channel_info->stream_id); send_output_bin = gst_bin_get_by_name(GST_BIN(priv->transport_bin), name); g_return_val_if_fail(send_output_bin, FALSE); g_free(name); @@ -3594,7 +3909,7 @@ static gboolean create_datachannel_appsrc(OwrTransportAgent *transport_agent, return result; } -static gboolean is_valid_sctp_stream_id(OwrTransportAgent *transport_agent, guint32 session_id, +static gboolean is_valid_sctp_session_id(OwrTransportAgent *transport_agent, guint32 session_id, guint16 sctp_stream_id, gboolean remotly_initiated) { gboolean is_client = FALSE; @@ -3712,12 +4027,13 @@ static void handle_data_channel_open_request(OwrTransportAgent *transport_agent, g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "Received data channel open request for data channel (%u)" ":\nordered = %u\nmax_packets_life_time = %d\nmax_packet_retransmits = %d\nprotocols = %s" - "\nnegotiated=%u\nlabel= %s\n", data_channel_info->id, data_channel_info->ordered, + "\nnegotiated=%u\nlabel= %s", data_channel_info->id, data_channel_info->ordered, data_channel_info->max_packet_life_time, data_channel_info->max_packet_retransmits, data_channel_info->protocol, data_channel_info->negotiated, data_channel_info->label); - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_object_ref(data_session); args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(data_session)); g_hash_table_insert(args, "session", data_session); @@ -3771,14 +4087,15 @@ static void handle_data_channel_ack(OwrTransportAgent *transport_agent, guint8 * OWR_UNUSED(data); OWR_UNUSED(size); - g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "Received ACK for data channel %u\n", sctp_stream_id); + g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "Received ACK for data channel %u", sctp_stream_id); g_rw_lock_reader_lock(&priv->data_channels_rw_mutex); data_channel_info = g_hash_table_lookup(priv->data_channels, GUINT_TO_POINTER(sctp_stream_id)); g_assert(data_channel_info); g_rw_lock_reader_unlock(&priv->data_channels_rw_mutex); g_rw_lock_writer_lock(&data_channel_info->rw_mutex); - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_assert(OWR_IS_DATA_SESSION(data_session)); data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); g_assert(OWR_IS_DATA_CHANNEL(data_channel)); @@ -3811,7 +4128,8 @@ static void handle_data_channel_message(OwrTransportAgent *transport_agent, guin goto end; } - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); owr_data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); g_assert(owr_data_channel); g_rw_lock_reader_unlock(&data_channel_info->rw_mutex); @@ -3877,7 +4195,7 @@ static void on_new_datachannel(OwrTransportAgent *transport_agent, OwrDataChanne remote_initiated = !!data_channel_info; if (!remote_initiated) { - guint session_id = get_stream_id(transport_agent, OWR_SESSION(data_session)); + guint session_id = get_session_id(transport_agent, OWR_SESSION(data_session)); if (!create_datachannel(transport_agent, session_id, data_channel)) { g_warning("Failed to create new datachannel"); goto end; @@ -3911,7 +4229,7 @@ static void on_new_datachannel(OwrTransportAgent *transport_agent, OwrDataChanne g_free(protocol); g_free(label); - g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "New data channel (%u) added\n", id); + g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "New data channel (%u) added", id); g_rw_lock_reader_unlock(&data_channel_info->rw_mutex); } @@ -4033,7 +4351,7 @@ static guint64 on_datachannel_request_bytes_sent(OwrTransportAgent *transport_ag DataChannel *data_channel_info; OwrDataSession *data_session; gchar *name; - guint ctrl_bytes_sent, session_id; + guint ctrl_bytes_sent, stream_id; g_object_get(data_channel, "id", &data_channel_id, NULL); g_rw_lock_reader_lock(&priv->data_channels_rw_mutex); @@ -4043,14 +4361,15 @@ static guint64 on_datachannel_request_bytes_sent(OwrTransportAgent *transport_ag g_rw_lock_reader_lock(&data_channel_info->rw_mutex); ctrl_bytes_sent = data_channel_info->ctrl_bytes_sent; - session_id = data_channel_info->session_id; + stream_id = data_channel_info->stream_id; g_rw_lock_reader_unlock(&data_channel_info->rw_mutex); - name = g_strdup_printf("send-output-bin-%u", session_id); + name = g_strdup_printf("send-output-bin-%u", stream_id); send_output_bin = gst_bin_get_by_name(GST_BIN(priv->transport_bin), name); g_free(name); - data_session = OWR_DATA_SESSION(get_session(transport_agent, session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + stream_id)); name = _owr_data_session_get_encoder_name(data_session); sctpenc = gst_bin_get_by_name(GST_BIN(send_output_bin), name); g_free(name); @@ -4075,8 +4394,8 @@ static void maybe_close_data_channel(OwrTransportAgent *transport_agent, OwrDataSession *data_session; OwrDataChannel *data_channel; - data_session = OWR_DATA_SESSION(get_session(transport_agent, - data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); _owr_data_channel_set_ready_state(data_channel, OWR_DATA_CHANNEL_READY_STATE_CLOSED); g_hash_table_steal(priv->data_channels, GUINT_TO_POINTER(data_channel_info->id)); @@ -4169,6 +4488,42 @@ gchar * owr_transport_agent_get_dot_data(OwrTransportAgent *transport_agent) #endif } +static gboolean dump_bin(gpointer data) +{ + GstPipeline* pipeline = GST_PIPELINE(data); + + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, GST_OBJECT_NAME(GST_OBJECT(pipeline))); + gst_object_unref(pipeline); + return G_SOURCE_REMOVE; +} + +void owr_transport_agent_start(OwrTransportAgent *agent) +{ + GSList *walk; + + if (!agent->priv->unstarted_sessions) + return; + + for (walk = agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { + GHashTable *args; + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(agent)); + g_hash_table_insert(args, "transport_agent", agent); + g_hash_table_insert(args, "session", g_object_ref(OWR_SESSION(walk->data))); + + g_object_ref(agent); + + add_session(args); + g_hash_table_unref(args); + } + g_slist_free_full(agent->priv->unstarted_sessions, g_object_unref); + agent->priv->unstarted_sessions = NULL; + + GstStateChangeReturn state_change_status; + state_change_status = gst_element_set_state(agent->priv->pipeline, GST_STATE_PLAYING); + g_warn_if_fail(state_change_status != GST_STATE_CHANGE_FAILURE); + + g_timeout_add_seconds(5, dump_bin, gst_object_ref(agent->priv->pipeline)); +} static void on_feedback_rtcp(GObject *session, guint type, guint fbtype, guint sender_ssrc, guint media_ssrc, GstBuffer *fci, OwrTransportAgent *transport_agent) @@ -4181,12 +4536,15 @@ static void on_feedback_rtcp(GObject *session, guint type, guint fbtype, guint s if (type == GST_RTCP_TYPE_RTPFB && fbtype == GST_RTCP_RTPFB_TYPE_SCREAM) { GstElement *send_output_bin, *scream_queue = NULL; GstMapInfo info = {NULL, 0, NULL, 0, 0, {0}, {0}}; /*GST_MAP_INFO_INIT;*/ - guint session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session_id")); + guint stream_id = GPOINTER_TO_UINT(g_object_get_data(session, "stream-id")); + guint session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session-id")); + gchar *queue_name = g_strdup_printf("screamqueue-%u-%u", session_id, stream_id); + gchar *name = g_strdup_printf("send-output-bin-%u", stream_id); - gchar *name = g_strdup_printf("send-output-bin-%u", session_id); send_output_bin = gst_bin_get_by_name(GST_BIN(transport_agent->priv->transport_bin), name); g_free(name); - scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), "screamqueue"); + scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), queue_name); + g_free(queue_name); gst_object_unref(send_output_bin); /* Read feedback from FCI */ @@ -4227,14 +4585,14 @@ static GstPadProbeReturn probe_save_ts(GstPad *srcpad, GstPadProbeInfo *info, vo static gint compare_rtcp_scream(GHashTable *a, GHashTable *b) { - guint session_id_a, session_id_b, ssrc_a, ssrc_b; + guint stream_id_a, stream_id_b, ssrc_a, ssrc_b; - session_id_a = GPOINTER_TO_UINT(g_hash_table_lookup(a, "session-id")); + stream_id_a = GPOINTER_TO_UINT(g_hash_table_lookup(a, "stream_id")); ssrc_a = GPOINTER_TO_UINT(g_hash_table_lookup(a, "ssrc")); - session_id_b = GPOINTER_TO_UINT(g_hash_table_lookup(b, "session-id")); + stream_id_b = GPOINTER_TO_UINT(g_hash_table_lookup(b, "stream_id")); ssrc_b = GPOINTER_TO_UINT(g_hash_table_lookup(b, "ssrc")); - if (session_id_a == session_id_b && ssrc_a == ssrc_b) + if (stream_id_a == stream_id_b && ssrc_a == ssrc_b) return 0; return -1; } @@ -4247,12 +4605,14 @@ static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, S guint64 arrival_time = GST_CLOCK_TIME_NONE; OwrTransportAgent *transport_agent = NULL; OwrTransportAgentPrivate *priv = NULL; + guint stream_id = 0; guint session_id = 0; guint8 pt = 0; gboolean rtp_mapped = FALSE; GObject *rtp_session = NULL; transport_agent = scream_rx->transport_agent; + stream_id = scream_rx->stream_id; session_id = scream_rx->session_id; g_assert(transport_agent); @@ -4270,21 +4630,38 @@ static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, S pt = gst_rtp_buffer_get_payload_type(&rtp_buf); } - g_signal_emit_by_name(priv->rtpbin, "get-internal-session", session_id, &rtp_session); + g_signal_emit_by_name(priv->rtpbin, "get-internal-session", stream_id, &rtp_session); if (G_UNLIKELY(scream_rx->rtx_pt == -2)) { - OwrMediaSession *media_session; - OwrPayload *rx_payload; + OwrMediaSession *media_session = NULL; + OwrPayload *rx_payload = NULL; OwrAdaptationType adapt_type; - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - rx_payload = _owr_media_session_get_receive_payload(media_session, pt); + + if (transport_agent->priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + rx_payload = _owr_media_session_get_receive_payload(media_session, pt); + } else { + GSList *sessions = get_sessions_from_stream_id(transport_agent, stream_id); + GSList *walk; + for (walk = sessions; walk; walk = g_slist_next(walk)) { + media_session = OWR_MEDIA_SESSION(walk->data); + rx_payload = _owr_media_session_get_receive_payload(media_session, pt); + if (rx_payload) + break; + } + g_slist_free_full(sessions, g_object_unref); + } + + g_assert(media_session); + g_assert(rx_payload); + g_object_get(rx_payload, "rtx-payload-type", &scream_rx->rtx_pt, "adaptation", &adapt_type, NULL); scream_rx->adapt = (adapt_type == OWR_ADAPTATION_TYPE_SCREAM); g_object_unref(media_session); g_object_unref(rx_payload); - g_object_set(rtp_session, "rtcp-reduced-size", TRUE, NULL); + g_object_set(rtp_session, "rtcp-reduced-size", TRUE, NULL); } OWR_UNUSED(srcpad); @@ -4368,7 +4745,7 @@ static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, S GUINT_TO_POINTER(scream_rx->highest_seq)); g_hash_table_insert(rtcp_info, "n-loss", GUINT_TO_POINTER(scream_rx->n_loss)); g_hash_table_insert(rtcp_info, "n-ecn", GUINT_TO_POINTER(scream_rx->n_ecn)); - g_hash_table_insert(rtcp_info, "session-id", GUINT_TO_POINTER(session_id)); + g_hash_table_insert(rtcp_info, "stream_id", GUINT_TO_POINTER(stream_id)); GST_LOG_OBJECT(transport_agent, "queuing up scream feedback: %u, %u, %u, %u", scream_rx->highest_seq, scream_rx->n_loss, scream_rx->n_ecn, diff --git a/transport/owr_transport_agent.h b/transport/owr_transport_agent.h index 950e4b4b..c149cea0 100644 --- a/transport/owr_transport_agent.h +++ b/transport/owr_transport_agent.h @@ -33,6 +33,7 @@ #define __OWR_TRANSPORT_AGENT_H__ #include "owr_session.h" +#include "owr_types.h" #include @@ -41,7 +42,8 @@ G_BEGIN_DECLS typedef enum _OwrHelperServerType { OWR_HELPER_SERVER_TYPE_STUN, OWR_HELPER_SERVER_TYPE_TURN_UDP, - OWR_HELPER_SERVER_TYPE_TURN_TCP + OWR_HELPER_SERVER_TYPE_TURN_TCP, + OWR_HELPER_SERVER_TYPE_TURN_TLS } OwrHelperServerType; #define OWR_TYPE_TRANSPORT_AGENT (owr_transport_agent_get_type()) @@ -69,7 +71,7 @@ struct _OwrTransportAgentClass { GType owr_transport_agent_get_type(void) G_GNUC_CONST; -OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode); +OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode, OwrBundlePolicyType bundle_policy_type); void owr_transport_agent_add_helper_server(OwrTransportAgent *transport_agent, OwrHelperServerType type, const gchar *address, guint port, const gchar *username, const gchar *password); void owr_transport_agent_add_local_address(OwrTransportAgent *transport_agent, const gchar *local_address); @@ -77,6 +79,8 @@ void owr_transport_agent_set_local_port_range(OwrTransportAgent *transport_agent void owr_transport_agent_add_session(OwrTransportAgent *agent, OwrSession *session); gchar * owr_transport_agent_get_dot_data(OwrTransportAgent *transport_agent); +void owr_transport_agent_start(OwrTransportAgent *agent); + G_END_DECLS #endif /* __OWR_TRANSPORT_AGENT_H__ */