//! Established ICE/DTLS-SRTP transport via GStreamer webrtcbin, with independent tracks. use super::{AudioFrame, VideoFrame}; use gst::prelude::*; use gstreamer as gst; use gstreamer_webrtc as rtc; use std::sync::{ Arc, Mutex, atomic::{AtomicU64, Ordering}, mpsc, }; #[derive(Clone, Copy, Debug, Default)] pub struct Tracks { pub application_audio: bool, pub microphone: bool, } #[derive(Debug)] pub enum Signal { Offer(String), Answer(String), Ice { mline: u32, candidate: String }, Error(String), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AudioTrack { Application, Microphone, } #[derive(Clone, Default)] pub struct RemoteFeed { video: Arc>>>, app: Arc>>, mic: Arc>>, pub video_frames: Arc, pub application_samples: Arc, pub microphone_samples: Arc, } impl RemoteFeed { pub fn take_video(&self) -> Option> { self.video.lock().ok()?.take() } pub fn take_audio(&self, track: AudioTrack) -> Option { match track { AudioTrack::Application => self.app.lock().ok()?.pop_front(), AudioTrack::Microphone => self.mic.lock().ok()?.pop_front(), } } } /// Explicit administrator-configured ICE services. Empty means host candidates only. #[derive(Clone, Debug, Default)] pub struct IceServers { pub stun: Option, pub turn: Option, } impl IceServers { pub fn validate(&self) -> Result<(), String> { if self.stun.as_ref().is_some_and(|uri| { !uri.starts_with("stun://") || uri.len() > 2048 || uri.len() <= 7 || uri.chars().any(char::is_whitespace) }) || self.turn.as_ref().is_some_and(|uri| { !(uri.starts_with("turn://") || uri.starts_with("turns://")) || uri.len() > 2048 || uri.len() <= 7 || uri.chars().any(char::is_whitespace) }) { return Err("Invalid explicitly configured ICE server URI".into()); } Ok(()) } } /// One directed publisher/viewer connection. The authenticated gateway routes its signals. pub struct Peer { pipeline: gst::Pipeline, rtc: gst::Element, video: Option, app: Option, mic: Option, events: mpsc::Receiver, tx: mpsc::SyncSender, pub remote: RemoteFeed, remote_set: bool, started: std::time::Instant, pending_ice: Vec<(u32, String)>, app_time: Mutex>, mic_time: Mutex>, } pub fn check_backend() -> Result<(), String> { gst::init().map_err(error)?; for plugin in [ "webrtcbin", "nicesrc", "nicesink", "dtlssrtpenc", "dtlssrtpdec", "vp8enc", "vp8dec", "rtpvp8pay", "rtpvp8depay", "opusenc", "opusdec", "rtpopuspay", "rtpopusdepay", ] { if gst::ElementFactory::find(plugin).is_none() { return Err(format!("Missing WebRTC plugin: {plugin}")); } } Ok(()) } impl Peer { pub fn publisher(tracks: Tracks) -> Result { Self::new(true, tracks, &IceServers::default()) } pub fn viewer(tracks: Tracks) -> Result { Self::new(false, tracks, &IceServers::default()) } pub fn publisher_with_ice(tracks: Tracks, ice: &IceServers) -> Result { Self::new(true, tracks, ice) } pub fn viewer_with_ice(tracks: Tracks, ice: &IceServers) -> Result { Self::new(false, tracks, ice) } fn new(publisher: bool, tracks: Tracks, ice: &IceServers) -> Result { ice.validate()?; check_backend()?; let pipeline = gst::Pipeline::new(); let rtc = gst::ElementFactory::make("webrtcbin") .name("transport") .build() .map_err(error)?; rtc.set_property_from_str("bundle-policy", "max-bundle"); rtc.set_property("latency", 100u32); if let Some(uri) = &ice.stun { rtc.set_property("stun-server", uri); } if let Some(uri) = &ice.turn { rtc.set_property("turn-server", uri); } pipeline.add(&rtc).map_err(error)?; let (tx, events) = mpsc::sync_channel(64); let ice_tx = tx.clone(); rtc.connect("on-ice-candidate", false, move |values| { if let (Ok(mline), Ok(candidate)) = (values[1].get::(), values[2].get::()) { let _ = ice_tx.try_send(Signal::Ice { mline, candidate }); } None }); let remote = RemoteFeed::default(); if !publisher { let weak_pipeline = pipeline.downgrade(); let feed = remote.clone(); let receive_tx = tx.clone(); rtc.connect_pad_added(move |_, pad| { if pad.direction() != gst::PadDirection::Src { return; } if let Some(pipeline) = weak_pipeline.upgrade() && let Err(error) = receive(&pipeline, pad, &feed, tracks) { let _ = receive_tx.try_send(Signal::Error(error)); } }); } let video = if publisher { Some(add_sender(&pipeline, &rtc, 96, "screen-video", true)?) } else { None }; let app = if publisher && tracks.application_audio { Some(add_sender(&pipeline, &rtc, 97, "application-audio", false)?) } else { None }; let mic = if publisher && tracks.microphone { Some(add_sender(&pipeline, &rtc, 98, "microphone", false)?) } else { None }; pipeline.set_state(gst::State::Playing).map_err(error)?; Ok(Self { pipeline, rtc, video, app, mic, events, tx, remote, remote_set: false, started: std::time::Instant::now(), pending_ice: Vec::new(), app_time: Mutex::new(None), mic_time: Mutex::new(None), }) } /// Create an SDP offer only after the gateway authorizes a viewer subscription. pub fn offer(&self) { create_description(&self.rtc, true, self.signal_sender()); } fn signal_sender(&self) -> mpsc::SyncSender { self.tx.clone() } pub fn apply_offer(&mut self, sdp: &str) -> Result<(), String> { self.set_description(sdp, true)?; create_description(&self.rtc, false, self.signal_sender()); Ok(()) } pub fn apply_answer(&mut self, sdp: &str) -> Result<(), String> { self.set_description(sdp, false) } fn set_description(&mut self, text: &str, offer: bool) -> Result<(), String> { if text.len() > 65_536 { return Err("SDP exceeds limit".into()); } let sdp = gstreamer_sdp::SDPMessage::parse_buffer(text.as_bytes()).map_err(error)?; let description = rtc::WebRTCSessionDescription::new( if offer { rtc::WebRTCSDPType::Offer } else { rtc::WebRTCSDPType::Answer }, sdp, ); let promise = gst::Promise::new(); self.rtc .emit_by_name::<()>("set-remote-description", &[&description, &promise]); if promise.wait() != gst::PromiseResult::Replied { return Err("Remote SDP was rejected".into()); } if let Some(reply) = promise.get_reply() && reply.has_field("error") { return Err("Remote SDP was rejected".into()); } self.remote_set = true; for (mline, candidate) in self.pending_ice.drain(..) { self.rtc .emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]); } Ok(()) } pub fn add_ice(&mut self, mline: u32, candidate: &str) -> Result<(), String> { if mline > 2 || candidate.len() > 2048 || self.pending_ice.len() >= 64 { return Err("ICE candidate exceeds limit".into()); } if self.remote_set { self.rtc .emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]); } else { self.pending_ice.push((mline, candidate.into())); } Ok(()) } pub fn poll(&self) -> Result, String> { if let Some(bus) = self.pipeline.bus() { while let Some(message) = bus.pop() { match message.view() { gst::MessageView::Error(e) => return Err(format!("WebRTC: {}", e.error())), gst::MessageView::Eos(..) => return Err("WebRTC stream closed".into()), _ => {} } } } let state = self .rtc .property::("connection-state"); if self.started.elapsed() > std::time::Duration::from_secs(20) && state != rtc::WebRTCPeerConnectionState::Connected { return Err("WebRTC connection timed out".into()); } if matches!( state, rtc::WebRTCPeerConnectionState::Failed | rtc::WebRTCPeerConnectionState::Closed ) { return Err("WebRTC peer disconnected".into()); } Ok(self.events.try_recv().ok()) } pub fn connected(&self) -> bool { self.rtc .property::("connection-state") == rtc::WebRTCPeerConnectionState::Connected } pub fn push_video(&self, frame: &VideoFrame) -> Result<(), String> { frame.validate()?; let source = self.video.as_ref().ok_or("Peer does not publish video")?; source.set_caps(Some( &gst::Caps::builder("video/x-raw") .field("format", "RGBA") .field("width", frame.width as i32) .field("height", frame.height as i32) .field("framerate", gst::Fraction::new(30, 1)) .build(), )); source .push_buffer(gst::Buffer::from_mut_slice(frame.rgba.clone())) .map_err(error)?; Ok(()) } pub fn push_audio(&self, track: AudioTrack, frame: &AudioFrame) -> Result<(), String> { frame.validate()?; let source = match track { AudioTrack::Application => &self.app, AudioTrack::Microphone => &self.mic, } .as_ref() .ok_or("Audio track was not explicitly enabled")?; source.set_caps(Some( &gst::Caps::builder("audio/x-raw") .field("format", "F32LE") .field("layout", "interleaved") .field("rate", frame.rate as i32) .field("channels", i32::from(frame.channels)) .build(), )); let bytes: Vec = frame .samples .iter() .flat_map(|sample| sample.to_le_bytes()) .collect(); let clock = match track { AudioTrack::Application => &self.app_time, AudioTrack::Microphone => &self.mic_time, }; let mut next = clock .lock() .map_err(|_| "Audio timestamp clock unavailable")?; let pts = next.unwrap_or_else(|| { self.pipeline .current_running_time() .map(|time| time.nseconds()) .unwrap_or(0) }); let duration = (frame.samples.len() as u64 / u64::from(frame.channels)) * 1_000_000_000 / u64::from(frame.rate); let mut buffer = gst::Buffer::from_mut_slice(bytes); if let Some(buffer) = buffer.get_mut() { buffer.set_pts(gst::ClockTime::from_nseconds(pts)); buffer.set_duration(gst::ClockTime::from_nseconds(duration)); } *next = Some(pts.saturating_add(duration)); source.push_buffer(buffer).map_err(error)?; Ok(()) } } impl Drop for Peer { fn drop(&mut self) { let _ = self.pipeline.set_state(gst::State::Null); } } fn add_sender( pipeline: &gst::Pipeline, rtc: &gst::Element, payload: i32, msid: &str, video: bool, ) -> Result { let description = if video { format!( "appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=8388608 leaky-type=downstream ! queue max-size-buffers=2 leaky=downstream ! videoconvert ! vp8enc deadline=1 cpu-used=8 target-bitrate=1500000 keyframe-max-dist=60 ! rtpvp8pay pt={payload} ! capsfilter caps=\"application/x-rtp,media=video,encoding-name=VP8,payload={payload},clock-rate=90000\"" ) } else { format!( "appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=384000 leaky-type=downstream ! queue max-size-buffers=8 leaky=downstream ! audioconvert ! audioresample ! opusenc bitrate=64000 ! rtpopuspay pt={payload} ! capsfilter caps=\"application/x-rtp,media=audio,encoding-name=OPUS,payload={payload},clock-rate=48000\"" ) }; let bin = gst::parse::bin_from_description(&description, true).map_err(error)?; let source = bin .by_name("source") .ok_or("Missing media appsrc")? .downcast::() .map_err(|_| "Wrong media source type")?; pipeline.add(&bin).map_err(error)?; let src = bin.static_pad("src").ok_or("Missing RTP source pad")?; let sink = rtc .request_pad_simple("sink_%u") .ok_or("WebRTC refused a media track")?; sink.set_property("msid", msid); let transceiver = sink.property::("transceiver"); transceiver.set_property("direction", rtc::WebRTCRTPTransceiverDirection::Sendonly); src.link(&sink).map_err(error)?; Ok(source) } fn create_description(webrtc: &gst::Element, offer: bool, tx: mpsc::SyncSender) { let weak = webrtc.downgrade(); let promise = gst::Promise::with_change_func(move |reply| { let result = (|| { let reply = reply .map_err(|e| format!("SDP promise failed: {e:?}"))? .ok_or("No SDP reply")?; let description = reply .get::(if offer { "offer" } else { "answer" }) .map_err(error)?; let webrtc = weak.upgrade().ok_or("WebRTC peer closed")?; webrtc.emit_by_name::<()>( "set-local-description", &[&description, &None::], ); let text = description.sdp().as_text().map_err(error)?; Ok::<_, String>(if offer { Signal::Offer(text) } else { Signal::Answer(text) }) })(); let _ = tx.try_send(result.unwrap_or_else(Signal::Error)); }); webrtc.emit_by_name::<()>( if offer { "create-offer" } else { "create-answer" }, &[&None::, &promise], ); } fn receive( pipeline: &gst::Pipeline, pad: &gst::Pad, feed: &RemoteFeed, tracks: Tracks, ) -> Result<(), String> { let caps = pad.current_caps().ok_or("Incoming track has no caps")?; let payload = caps .structure(0) .ok_or("Incoming track has no format")? .get::("payload") .map_err(error)?; let video = payload == 96; if !(video || payload == 97 && tracks.application_audio || payload == 98 && tracks.microphone) { return Err("Unannounced remote track rejected".into()); } let description = if video { "queue max-size-buffers=2 leaky=downstream ! rtpvp8depay ! vp8dec ! videoconvert ! video/x-raw,format=RGBA,width=[1,1920],height=[1,1080] ! appsink name=sink sync=false max-buffers=1 drop=true" } else { "queue max-size-buffers=8 leaky=downstream ! rtpopusdepay ! opusdec ! audioconvert ! audioresample ! audio/x-raw,format=F32LE,layout=interleaved,rate=48000,channels=2 ! appsink name=sink sync=false max-buffers=4 drop=true" }; let bin = gst::parse::bin_from_description(description, true).map_err(error)?; let sink = bin .by_name("sink") .ok_or("Remote sink missing")? .downcast::() .map_err(|_| "Wrong remote sink")?; let feed = feed.clone(); sink.set_callbacks( gstreamer_app::AppSinkCallbacks::builder() .new_sample(move |sink| { let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?; if video { let frame = super::capture::copy_video(&sample).map_err(|_| gst::FlowError::Error)?; *feed.video.lock().map_err(|_| gst::FlowError::Error)? = Some(Arc::new(frame)); feed.video_frames.fetch_add(1, Ordering::Relaxed); } else { let map = sample .buffer() .ok_or(gst::FlowError::Error)? .map_readable() .map_err(|_| gst::FlowError::Error)?; if map.len() > 384000 { return Err(gst::FlowError::Error); } let samples: Vec = map .as_slice() .as_chunks::<4>() .0 .iter() .map(|bytes| f32::from_le_bytes(*bytes)) .collect(); let count = samples.len() as u64; let frame = AudioFrame { samples, rate: 48000, channels: 2, }; frame.validate().map_err(|_| gst::FlowError::Error)?; let (queue, received) = if payload == 97 { (&feed.app, &feed.application_samples) } else { (&feed.mic, &feed.microphone_samples) }; let mut queue = queue.lock().map_err(|_| gst::FlowError::Error)?; if queue.len() == 8 { queue.pop_front(); } queue.push_back(frame); received.fetch_add(count, Ordering::Relaxed); } Ok(gst::FlowSuccess::Ok) }) .build(), ); pipeline.add(&bin).map_err(error)?; bin.sync_state_with_parent().map_err(error)?; pad.link(&bin.static_pad("sink").ok_or("Missing remote RTP sink")?) .map_err(error)?; Ok(()) } fn error(error: impl std::fmt::Display) -> String { error.to_string() } #[cfg(test)] mod tests { use super::*; use std::time::{Duration, Instant}; fn forward(from: &Peer, to: &mut Peer) { while let Some(signal) = from.poll().expect("healthy peer") { match signal { Signal::Offer(sdp) => to.apply_offer(&sdp).expect("offer accepted"), Signal::Answer(sdp) => to.apply_answer(&sdp).expect("answer accepted"), Signal::Ice { mline, candidate } => { to.add_ice(mline, &candidate).expect("candidate accepted") } Signal::Error(error) => panic!("signalling failed: {error}"), } } } #[test] fn two_peers_deliver_video_and_separate_application_and_microphone_tracks() { let tracks = Tracks { application_audio: true, microphone: true, }; let mut publisher = Peer::publisher(tracks).expect("publisher"); let mut viewer = Peer::viewer(tracks).expect("viewer"); let frame = VideoFrame { width: 64, height: 48, rgba: [200, 50, 70, 255].repeat(64 * 48), timestamp_ns: None, }; publisher.push_video(&frame).expect("fixture video"); publisher.offer(); let start = Instant::now(); let mut index = 0usize; let mut app_pcm = Vec::new(); let mut mic_pcm = Vec::new(); while start.elapsed() < Duration::from_secs(12) { forward(&publisher, &mut viewer); forward(&viewer, &mut publisher); publisher.push_video(&frame).expect("video push"); for (track, frequency) in [ (AudioTrack::Application, 440.0f32), (AudioTrack::Microphone, 880.0f32), ] { let samples = (index..index + 960) .flat_map(|sample| { let value = (sample as f32 * frequency * std::f32::consts::TAU / 48_000.0) .sin() * 0.3; [value, value] }) .collect(); publisher .push_audio( track, &AudioFrame { samples, rate: 48_000, channels: 2, }, ) .expect("independent audio push"); } index += 960; while let Some(frame) = viewer.remote.take_audio(AudioTrack::Application) { app_pcm.extend(frame.samples.into_iter().step_by(2)); } while let Some(frame) = viewer.remote.take_audio(AudioTrack::Microphone) { mic_pcm.extend(frame.samples.into_iter().step_by(2)); } if viewer.remote.video_frames.load(Ordering::Relaxed) >= 3 && app_pcm.len() > 24_000 && mic_pcm.len() > 24_000 { break; } std::thread::sleep(Duration::from_millis(20)); } assert!( publisher.connected() && viewer.connected(), "DTLS/ICE connection established" ); let received = viewer .remote .take_video() .expect("actual decoded remote video"); assert_eq!((received.width, received.height), (64, 48)); assert!((i32::from(received.rgba[0]) - 200).abs() < 20); assert!( app_pcm.len() > 24_000 && mic_pcm.len() > 24_000, "both distinct Opus tracks delivered" ); let frequency = |pcm: &[f32]| { pcm.windows(2) .filter(|pair| pair[0] < 0.0 && pair[1] >= 0.0) .count() as f32 * 48_000.0 / pcm.len() as f32 }; let app_frequency = frequency(&app_pcm); let mic_frequency = frequency(&mic_pcm); assert!( (app_frequency - 440.0).abs() < 60.0, "app frequency {app_frequency}" ); assert!( (mic_frequency - 880.0).abs() < 100.0, "microphone frequency {mic_frequency}" ); assert!( publisher .push_audio( AudioTrack::Application, &AudioFrame { samples: vec![f32::NAN], rate: 48_000, channels: 1 } ) .is_err() ); } #[test] fn unrequested_audio_track_cannot_be_enabled_by_push() { let publisher = Peer::publisher(Tracks::default()).expect("publisher"); let pcm = AudioFrame { samples: vec![0.0; 960], rate: 48_000, channels: 1, }; assert!(publisher.push_audio(AudioTrack::Microphone, &pcm).is_err()); assert!(publisher.push_audio(AudioTrack::Application, &pcm).is_err()); } }