Custom video sink

webrtc-jni/pom.xml

@@ -12,7 +12,7 @@
 	<packaging>pom</packaging>
 
 	<properties>
-		<webrtc.branch>branch-heads/4844</webrtc.branch>
+		<webrtc.branch>branch-heads/5572</webrtc.branch>
 		<webrtc.src.dir>${user.home}/webrtc</webrtc.src.dir>
 		<webrtc.install.dir>${user.home}/webrtc/build</webrtc.install.dir>
 		<cmake.build.type>Release</cmake.build.type>

webrtc-jni/src/main/cpp/CMakeLists.txt

@@ -55,13 +55,19 @@ target_include_directories(${PROJECT_NAME}
 		include/media
 		include/rtc
 )
-
-set_target_properties(${PROJECT_NAME} PROPERTIES
-	CXX_STANDARD 17
-	CXX_STANDARD_REQUIRED ON
-	CXX_EXTENSIONS OFF
-)
-
+if(LINUX)
+    set_target_properties(${PROJECT_NAME} PROPERTIES
+        CXX_STANDARD 17
+        CXX_STANDARD_REQUIRED ON
+        CXX_EXTENSIONS OFF
+    )
+elseif(WIN32)
+    set_target_properties(${PROJECT_NAME} PROPERTIES
+        CXX_STANDARD 14
+        CXX_STANDARD_REQUIRED ON
+        CXX_EXTENSIONS OFF
+    )
+endif()
 target_link_libraries(${PROJECT_NAME} jni-voithos)
 target_link_libraries(${PROJECT_NAME} webrtc)
 
@@ -77,4 +83,4 @@ endif()
 install(TARGETS ${PROJECT_NAME}
 	RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT Runtime
 	LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT Runtime
-)
+)

webrtc-jni/src/main/cpp/dependencies/jni-voithos/include/Exception.h

@@ -10,6 +10,7 @@
 
 #include <stdarg.h>
 #include <string>
+#include <stdint.h>
 
 #ifndef _MSC_VER
 #define NOEXCEPT noexcept

webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt

@@ -203,7 +203,7 @@ if (PATCHES)
 endif()
 
 message(STATUS "WebRTC: generate")
-set(COMPILE_ARGS "is_debug=false target_cpu=\"${TARGET_CPU}\" treat_warnings_as_errors=false rtc_build_examples=false rtc_include_tests=false use_rtti=true use_custom_libcxx=false symbol_level=0 rtc_use_h264=true")
+set(COMPILE_ARGS "is_debug=false target_cpu=\"${TARGET_CPU}\" treat_warnings_as_errors=false rtc_build_examples=false rtc_include_tests=false use_rtti=true use_custom_libcxx=false symbol_level=0 rtc_use_h264=true proprietary_codecs=true use_lld=false grpc_use_static_linking=true")
 execute_command(
 	COMMAND gn gen ${WEBRTC_BUILD} --args=${COMPILE_ARGS}
 	WORKING_DIRECTORY "${WEBRTC_SRC}"

webrtc-jni/src/main/cpp/include/JNI_CustomVideoSource.h

@@ -0,0 +1,18 @@
+//
+// Created by kalgecin on 10/26/23.
+//
+#include <jni.h>
+
+#ifndef WEBRTC_JAVA_JNI_CUSTOMVIDEOSOURCE_H
+#define WEBRTC_JAVA_JNI_CUSTOMVIDEOSOURCE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+    JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_initialize
+            (JNIEnv * env, jobject caller);
+    JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_OnFrameCaptured
+            (JNIEnv *, jobject, jobject);
+#ifdef __cplusplus
+}
+#endif
+#endif //WEBRTC_JAVA_JNI_CUSTOMVIDEOSOURCE_H

webrtc-jni/src/main/cpp/include/media/audio/AudioConverter.h

@@ -22,7 +22,7 @@
 
 #include <jni.h>
 
-#include "rtc_base/constructor_magic.h"
+#include "third_party/breakpad/breakpad/src/common/basictypes.h"
 
 namespace jni
 {
@@ -52,7 +52,7 @@ namespace jni
 			const size_t dstChannels;
 
 		private:
-			RTC_DISALLOW_COPY_AND_ASSIGN(AudioConverter);
+        DISALLOW_COPY_AND_ASSIGN(AudioConverter);
 	};
 }
 

webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h

@@ -0,0 +1,24 @@
+//
+// Created by kalgecin on 10/25/23.
+//
+
+#ifndef WEBRTC_JAVA_CUSTOMVIDEOSOURCE_H
+#define WEBRTC_JAVA_CUSTOMVIDEOSOURCE_H
+
+#include <media/base/adapted_video_track_source.h>
+
+namespace jni {
+    class CustomVideoSource : public rtc::AdaptedVideoTrackSource {
+    public:
+        void OnFrameCaptured(const webrtc::VideoFrame &frame);
+
+        bool remote() const override;
+
+        SourceState state() const override;
+
+        bool is_screencast() const override;
+
+        absl::optional<bool> needs_denoising() const override;
+    };
+}
+#endif //WEBRTC_JAVA_CUSTOMVIDEOSOURCE_H

webrtc-jni/src/main/cpp/src/JNI_CustomVideoSource.cpp

@@ -0,0 +1,25 @@
+//
+// Created by kalgecin on 10/26/23.
+//
+
+#include "JNI_CustomVideoSource.h"
+#include "media/video//CustomVideoSource.h"
+#include "VideoFrame.h"
+
+JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_OnFrameCaptured
+        (JNIEnv *env, jobject obj, jobject videoFrame) {
+    jclass c = env->GetObjectClass(obj);
+    jfieldID fid_handle = GetHandleField(env, obj, "nativeHandle");
+    auto * nativeObject = (jni::CustomVideoSource *) env->GetLongField(obj, fid_handle);
+
+    const webrtc::VideoFrame &nativeFrame = jni::VideoFrame::toNative(env, jni::JavaLocalRef<jobject>(env, videoFrame));
+    nativeObject->OnFrameCaptured(nativeFrame);
+}
+
+JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_initialize
+        (JNIEnv * env, jobject caller)
+{
+    rtc::scoped_refptr<jni::CustomVideoSource> videoSource = rtc::scoped_refptr<jni::CustomVideoSource>(new rtc::RefCountedObject<jni::CustomVideoSource>());
+
+    SetHandle(env, caller, videoSource.release());
+}

webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp

@@ -66,7 +66,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize
 			networkThread.get(),
 			workerThread.get(),
 			signalingThread.get(),
-			audioDevModule,
+            rtc::scoped_refptr<webrtc::AudioDeviceModule>(audioDevModule),
 			webrtc::CreateBuiltinAudioEncoderFactory(),
 			webrtc::CreateBuiltinAudioDecoderFactory(),
 			webrtc::CreateBuiltinVideoEncoderFactory(),

webrtc-jni/src/main/cpp/src/JNI_RTCPeerConnection.cpp

@@ -119,7 +119,7 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_RTCPeerConnection_addTrack
 
 	std::vector<std::string> streamIDs = jni::JavaList::toStringVector(env, jni::JavaLocalRef<jobject>(env, jStreamIds));
 
-	auto result = pc->AddTrack(track, streamIDs);
+	auto result = pc->AddTrack(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>(track), streamIDs);
 
 	if (result.ok()) {
 		auto sender = result.MoveValue();
@@ -146,7 +146,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCPeerConnection_removeTrack
 	webrtc::RtpSenderInterface * sender = GetHandle<webrtc::RtpSenderInterface>(env, jSender);
 	CHECK_HANDLE(sender);
 
-	auto result = pc->RemoveTrackOrError(sender);
+	auto result = pc->RemoveTrackOrError(rtc::scoped_refptr<webrtc::RtpSenderInterface>(sender));
 
 	if (!result.ok()) {
 		env->Throw(jni::JavaRuntimeException(env, "Remove track (RTCRtpSender) failed: %s %s",
@@ -173,10 +173,10 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_RTCPeerConnection_addTransceive
 	if (jTransceiverInit != nullptr) {
 		auto init = jni::RTCRtpTransceiverInit::toNative(env, jni::JavaLocalRef<jobject>(env, jTransceiverInit));
 
-		result = pc->AddTransceiver(track, init);
+		result = pc->AddTransceiver(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>(track), init);
 	}
 	else {
-		result = pc->AddTransceiver(track);
+		result = pc->AddTransceiver(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>(track));
 	}
 
 	if (result.ok()) {
@@ -556,7 +556,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCPeerConnection_getStats__Ldev_o
 
 	auto callback = new rtc::RefCountedObject<jni::RTCStatsCollectorCallback>(env, jni::JavaGlobalRef<jobject>(env, jcallback));
 
-	pc->GetStats(receiver, callback);
+	pc->GetStats(rtc::scoped_refptr<webrtc::RtpReceiverInterface>(receiver), rtc::scoped_refptr<webrtc::RTCStatsCollectorCallback>(callback));
 }
 
 JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCPeerConnection_getStats__Ldev_onvoid_webrtc_RTCRtpSender_2Ldev_onvoid_webrtc_RTCStatsCollectorCallback_2
@@ -579,7 +579,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCPeerConnection_getStats__Ldev_o
 
 	auto callback = new rtc::RefCountedObject<jni::RTCStatsCollectorCallback>(env, jni::JavaGlobalRef<jobject>(env, jcallback));
 
-	pc->GetStats(sender, callback);
+	pc->GetStats(rtc::scoped_refptr<webrtc::RtpReceiverInterface>(sender), rtc::scoped_refptr<webrtc::RTCStatsCollectorCallback>(callback));
 }
 
 JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_RTCPeerConnection_restartIce

webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp

@@ -108,7 +108,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_dis
 JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_initialize
 (JNIEnv * env, jobject caller)
 {
-	rtc::scoped_refptr<jni::VideoTrackDesktopSource> videoSource = new rtc::RefCountedObject<jni::VideoTrackDesktopSource>();
+	rtc::scoped_refptr<jni::VideoTrackDesktopSource> videoSource = rtc::scoped_refptr<jni::VideoTrackDesktopSource>(new rtc::RefCountedObject<jni::VideoTrackDesktopSource>());
 
 	SetHandle(env, caller, videoSource.release());
 }

webrtc-jni/src/main/cpp/src/JNI_VideoDeviceSource.cpp

@@ -110,7 +110,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_disp
 JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_initialize
 (JNIEnv * env, jobject caller)
 {
-	rtc::scoped_refptr<jni::VideoTrackDeviceSource> videoSource = new rtc::RefCountedObject<jni::VideoTrackDeviceSource>();
+	rtc::scoped_refptr<jni::VideoTrackDeviceSource> videoSource = rtc::scoped_refptr<jni::VideoTrackDeviceSource>(new rtc::RefCountedObject<jni::VideoTrackDeviceSource>());
 
 	SetHandle(env, caller, videoSource.release());
 }

webrtc-jni/src/main/cpp/src/api/AudioOptions.cpp

@@ -35,8 +35,17 @@ namespace jni
 			options.auto_gain_control = obj.getBoolean(javaClass->autoGainControl);
 			options.noise_suppression = obj.getBoolean(javaClass->noiseSuppression);
 			options.highpass_filter = obj.getBoolean(javaClass->highpassFilter);
-			options.typing_detection = obj.getBoolean(javaClass->typingDetection);
-			options.residual_echo_detector = obj.getBoolean(javaClass->residualEchoDetector);
+
+            // Deprecated.
+            // TODO(bugs.webrtc.org/11226): Remove.
+            // Audio processing to detect typing.
+            absl::optional<bool> typing_detection;
+//			options.typing_detection = obj.getBoolean(javaClass->typingDetection); //todo
+
+            // TODO(bugs.webrtc.org/11539): Deprecated, replaced by
+            // webrtc::CreateEchoDetector() and injection when creating the audio
+            // processing module.
+//			options.residual_echo_detector = obj.getBoolean(javaClass->residualEchoDetector); todo
 
 			return options;
 		}

webrtc-jni/src/main/cpp/src/api/RTCConfiguration.cpp

@@ -75,7 +75,12 @@ namespace jni
 
 			configuration.servers = JavaList::toVector(env, is, &RTCIceServer::toNative);
 			configuration.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
-			configuration.enable_dtls_srtp = true;
+            // TODO(bugs.webrtc.org/11066): Remove entirely once Fuchsia does not use.
+            // TODO(bugs.webrtc.org/9891) - Move to crypto_options
+            // Can be used to disable DTLS-SRTP. This should never be done, but can be
+            // useful for testing purposes, for example in setting up a loopback call
+            // with a single PeerConnection.
+//			configuration.enable_dtls_srtp = true; //todo
 			configuration.type = JavaEnums::toNative<webrtc::PeerConnectionInterface::IceTransportsType>(env, tp);
 			configuration.bundle_policy = JavaEnums::toNative<webrtc::PeerConnectionInterface::BundlePolicy>(env, bp);
 			configuration.rtcp_mux_policy = JavaEnums::toNative<webrtc::PeerConnectionInterface::RtcpMuxPolicy>(env, mp);

webrtc-jni/src/main/cpp/src/api/RTCRtpContributingSource.cpp

@@ -49,8 +49,8 @@ namespace jni
 				static_cast<int64_t>(obj.getLong(javaClass->timestamp)),
 				static_cast<uint32_t>(obj.getLong(javaClass->source)),
 				webrtc::RtpSourceType::CSRC,
-				static_cast<uint8_t>(obj.getDouble(javaClass->audioLevel)),
-				static_cast<uint32_t>(obj.getLong(javaClass->rtpTimestamp))
+				static_cast<uint32_t>(obj.getLong(javaClass->rtpTimestamp)),
+                {static_cast<uint8_t>(obj.getDouble(javaClass->audioLevel)), absl::nullopt}
 			);
 		}
 

webrtc-jni/src/main/cpp/src/api/RTCRtpSynchronizationSource.cpp

@@ -48,13 +48,13 @@ namespace jni
 
 			JavaObject obj(env, source);
 
-			return webrtc::RtpSource(
-				static_cast<int64_t>(obj.getLong(parentClass->timestamp)),
-				static_cast<uint32_t>(obj.getLong(parentClass->source)),
-				webrtc::RtpSourceType::SSRC,
-				static_cast<uint8_t>(obj.getDouble(parentClass->audioLevel)),
-				static_cast<uint32_t>(obj.getLong(parentClass->rtpTimestamp))
-			);
+            return webrtc::RtpSource(
+                    static_cast<int64_t>(obj.getLong(parentClass->timestamp)),
+                    static_cast<uint32_t>(obj.getLong(parentClass->source)),
+                    webrtc::RtpSourceType::CSRC,
+                    static_cast<uint32_t>(obj.getLong(parentClass->rtpTimestamp)),
+                    {static_cast<uint8_t>(obj.getDouble(parentClass->audioLevel)), absl::nullopt}
+            );
 		}
 
 		JavaRTCRtpSynchronizationSourceClass::JavaRTCRtpSynchronizationSourceClass(JNIEnv * env)

webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+#include <api/video/i420_buffer.h>
 #include "api/VideoFrame.h"
 #include "JavaObject.h"
 #include "JavaUtils.h"
@@ -32,10 +33,24 @@ namespace jni
 
 			int rotation = obj.getInt(javaClass->rotation);
 			int64_t timestamp_ns = obj.getLong(javaClass->timestampNs);
+            const auto jBuffer = JavaClasses::get<JavaNativeI420BufferClass>(env);
+            JavaObject objBuffer(env, obj.getObject(javaClass->buffer));
 
+            const uint8_t * src_y = static_cast<uint8_t *>(env->GetDirectBufferAddress(objBuffer.getObject(jBuffer->dataY).get()));
+            const uint8_t * src_u = static_cast<uint8_t *>(env->GetDirectBufferAddress(objBuffer.getObject(jBuffer->dataU).get()));
+            const uint8_t * src_v = static_cast<uint8_t *>(env->GetDirectBufferAddress(objBuffer.getObject(jBuffer->dataV).get()));
+
+            int stride_y = static_cast<int32_t>(objBuffer.getInt(jBuffer->strideY));
+            int stride_u = static_cast<int32_t>(objBuffer.getInt(jBuffer->strideU));
+            int stride_v = static_cast<int32_t>(objBuffer.getInt(jBuffer->strideV));
+
+            int width = static_cast<int32_t>(objBuffer.getInt(jBuffer->width));
+            int height = static_cast<int32_t>(objBuffer.getInt(jBuffer->height));
+
+            const rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = webrtc::I420Buffer::Copy(width,height,src_y,stride_y,src_u,stride_u,src_v,stride_v);
 			return webrtc::VideoFrame::Builder()
-				//.set_video_frame_buffer(buffer)
-				//.set_timestamp_rtp(timestamp_rtp)
+				.set_video_frame_buffer(buffer)
+				.set_timestamp_rtp(timestamp_ns)
 				.set_timestamp_ms(timestamp_ns / rtc::kNumNanosecsPerMillisec)
 				.set_rotation(static_cast<webrtc::VideoRotation>(rotation))
 				.build();

webrtc-jni/src/main/cpp/src/media/audio/AudioProcessingConfig.cpp

@@ -61,12 +61,13 @@ namespace jni
 			if (nsLevel.get()) {
 				config.noise_suppression.level = jni::JavaEnums::toNative<webrtc::AudioProcessing::Config::NoiseSuppression::Level>(env, nsLevel);
 			}
-
-			config.residual_echo_detector.enabled = residualEchoDetector.getBoolean(javaResidualEchoDetectorClass->enabled);
+            // TODO(bugs.webrtc.org/11539): Deprecated. Delete this flag. Replaced by
+            // injectable submodule.
+//			config.residual_echo_detector.enabled = residualEchoDetector.getBoolean(javaResidualEchoDetectorClass->enabled); todo
 
 			config.transient_suppression.enabled = transientSuppression.getBoolean(javaTransientSuppressionClass->enabled);
 
-			config.voice_detection.enabled = voiceDetection.getBoolean(javaVoiceDetectionClass->enabled);
+//			config.voice_detection.enabled = voiceDetection.getBoolean(javaVoiceDetectionClass->enabled); todo
 
 			return config;
 		}
@@ -86,9 +87,14 @@ namespace jni
 			gainController.enabled = gainControl.getBoolean(javaGainControlClass->enabled);
 			gainController.fixed_digital.gain_db = gainControlFixedDigital.getFloat(javaGainControlFixedDigitalClass->gainDb);
 			gainController.adaptive_digital.enabled = gainControlAdaptiveDigital.getBoolean(javaGainControlAdaptiveDigitalClass->enabled);
-			gainController.adaptive_digital.dry_run = gainControlAdaptiveDigital.getBoolean(javaGainControlAdaptiveDigitalClass->dryRun);
-			gainController.adaptive_digital.vad_reset_period_ms = gainControlAdaptiveDigital.getInt(javaGainControlAdaptiveDigitalClass->vadResetPeriodMs);
-			gainController.adaptive_digital.adjacent_speech_frames_threshold = gainControlAdaptiveDigital.getInt(javaGainControlAdaptiveDigitalClass->adjacentSpeechFramesThreshold);
+            // TODO(bugs.webrtc.org/7494): Remove `dry_run`.
+            // When true, the adaptive digital controller runs but the signal is not
+            // modified. dfba28e30eaa791147c98e34ef0476e99eb93f5e
+//			gainController.adaptive_digital.dry_run = gainControlAdaptiveDigital.getBoolean(javaGainControlAdaptiveDigitalClass->dryRun); todo
+            // TODO(bugs.webrtc.org/7494): Hard-code and remove parameter below. dfba28e30eaa791147c98e34ef0476e99eb93f5e
+//			gainController.adaptive_digital.vad_reset_period_ms = gainControlAdaptiveDigital.getInt(javaGainControlAdaptiveDigitalClass->vadResetPeriodMs); todo
+            // TODO(bugs.webrtc.org/7494): Hard-code and remove parameter below. dfba28e30eaa791147c98e34ef0476e99eb93f5e
+//			gainController.adaptive_digital.adjacent_speech_frames_threshold = gainControlAdaptiveDigital.getInt(javaGainControlAdaptiveDigitalClass->adjacentSpeechFramesThreshold); todo
 			gainController.adaptive_digital.max_gain_change_db_per_second = gainControlAdaptiveDigital.getFloat(javaGainControlAdaptiveDigitalClass->maxGainChangeDbPerSecond);
 			gainController.adaptive_digital.max_output_noise_level_dbfs = gainControlAdaptiveDigital.getFloat(javaGainControlAdaptiveDigitalClass->maxOutputNoiseLevelDbfs);
 

webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp

@@ -0,0 +1,29 @@
+//
+// Created by kalgecin on 10/25/23.
+//
+
+#include "video/CustomVideoSource.h"
+#include "rtc_base/logging.h"
+
+namespace jni {
+    void CustomVideoSource::OnFrameCaptured(const webrtc::VideoFrame& frame) {
+        RTC_LOG(LS_VERBOSE) << "VideoFrame: " << frame.width() << "x" << frame.height() << " timestamp: " << frame.timestamp_us();
+        OnFrame(frame);
+    }
+
+    webrtc::MediaSourceInterface::SourceState CustomVideoSource::state() const {
+        return kLive;
+    }
+
+    bool CustomVideoSource::remote() const {
+        return false;
+    }
+
+    bool CustomVideoSource::is_screencast() const {
+        return false;
+    }
+
+    absl::optional<bool> CustomVideoSource::needs_denoising() const {
+        return false;
+    }
+}