VNOX-Client/src/media/peer.rs
2026-10-24 16:51:15 +02:00

646 lines
24 KiB
Rust

//! Established ICE/DTLS-SRTP transport via GStreamer webrtcbin, with independent tracks.
use super::{AudioFrame, VideoFrame};
use gst::prelude::*;
use gstreamer as gst;
use gstreamer_webrtc as rtc;
use std::sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
mpsc,
};
#[derive(Clone, Copy, Debug, Default)]
pub struct Tracks {
pub application_audio: bool,
pub microphone: bool,
}
#[derive(Debug)]
pub enum Signal {
Offer(String),
Answer(String),
Ice { mline: u32, candidate: String },
Error(String),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AudioTrack {
Application,
Microphone,
}
#[derive(Clone, Default)]
pub struct RemoteFeed {
video: Arc<Mutex<Option<Arc<VideoFrame>>>>,
app: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
mic: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
pub video_frames: Arc<AtomicU64>,
pub application_samples: Arc<AtomicU64>,
pub microphone_samples: Arc<AtomicU64>,
}
impl RemoteFeed {
pub fn take_video(&self) -> Option<Arc<VideoFrame>> {
self.video.lock().ok()?.take()
}
pub fn take_audio(&self, track: AudioTrack) -> Option<AudioFrame> {
match track {
AudioTrack::Application => self.app.lock().ok()?.pop_front(),
AudioTrack::Microphone => self.mic.lock().ok()?.pop_front(),
}
}
}
/// Explicit administrator-configured ICE services. Empty means host candidates only.
#[derive(Clone, Debug, Default)]
pub struct IceServers {
pub stun: Option<String>,
pub turn: Option<String>,
}
impl IceServers {
pub fn validate(&self) -> Result<(), String> {
if self.stun.as_ref().is_some_and(|uri| {
!uri.starts_with("stun://")
|| uri.len() > 2048
|| uri.len() <= 7
|| uri.chars().any(char::is_whitespace)
}) || self.turn.as_ref().is_some_and(|uri| {
!(uri.starts_with("turn://") || uri.starts_with("turns://"))
|| uri.len() > 2048
|| uri.len() <= 7
|| uri.chars().any(char::is_whitespace)
}) {
return Err("Invalid explicitly configured ICE server URI".into());
}
Ok(())
}
}
/// One directed publisher/viewer connection. The authenticated gateway routes its signals.
pub struct Peer {
pipeline: gst::Pipeline,
rtc: gst::Element,
video: Option<gstreamer_app::AppSrc>,
app: Option<gstreamer_app::AppSrc>,
mic: Option<gstreamer_app::AppSrc>,
events: mpsc::Receiver<Signal>,
tx: mpsc::SyncSender<Signal>,
pub remote: RemoteFeed,
remote_set: bool,
started: std::time::Instant,
pending_ice: Vec<(u32, String)>,
app_time: Mutex<Option<u64>>,
mic_time: Mutex<Option<u64>>,
}
pub fn check_backend() -> Result<(), String> {
gst::init().map_err(error)?;
for plugin in [
"webrtcbin",
"nicesrc",
"nicesink",
"dtlssrtpenc",
"dtlssrtpdec",
"vp8enc",
"vp8dec",
"rtpvp8pay",
"rtpvp8depay",
"opusenc",
"opusdec",
"rtpopuspay",
"rtpopusdepay",
] {
if gst::ElementFactory::find(plugin).is_none() {
return Err(format!("Missing WebRTC plugin: {plugin}"));
}
}
Ok(())
}
impl Peer {
pub fn publisher(tracks: Tracks) -> Result<Self, String> {
Self::new(true, tracks, &IceServers::default())
}
pub fn viewer(tracks: Tracks) -> Result<Self, String> {
Self::new(false, tracks, &IceServers::default())
}
pub fn publisher_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
Self::new(true, tracks, ice)
}
pub fn viewer_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
Self::new(false, tracks, ice)
}
fn new(publisher: bool, tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
ice.validate()?;
check_backend()?;
let pipeline = gst::Pipeline::new();
let rtc = gst::ElementFactory::make("webrtcbin")
.name("transport")
.build()
.map_err(error)?;
rtc.set_property_from_str("bundle-policy", "max-bundle");
rtc.set_property("latency", 100u32);
if let Some(uri) = &ice.stun {
rtc.set_property("stun-server", uri);
}
if let Some(uri) = &ice.turn {
rtc.set_property("turn-server", uri);
}
pipeline.add(&rtc).map_err(error)?;
let (tx, events) = mpsc::sync_channel(64);
let ice_tx = tx.clone();
rtc.connect("on-ice-candidate", false, move |values| {
if let (Ok(mline), Ok(candidate)) = (values[1].get::<u32>(), values[2].get::<String>())
{
let _ = ice_tx.try_send(Signal::Ice { mline, candidate });
}
None
});
let remote = RemoteFeed::default();
if !publisher {
let weak_pipeline = pipeline.downgrade();
let feed = remote.clone();
let receive_tx = tx.clone();
rtc.connect_pad_added(move |_, pad| {
if pad.direction() != gst::PadDirection::Src {
return;
}
if let Some(pipeline) = weak_pipeline.upgrade()
&& let Err(error) = receive(&pipeline, pad, &feed, tracks)
{
let _ = receive_tx.try_send(Signal::Error(error));
}
});
}
let video = if publisher {
Some(add_sender(&pipeline, &rtc, 96, "screen-video", true)?)
} else {
None
};
let app = if publisher && tracks.application_audio {
Some(add_sender(&pipeline, &rtc, 97, "application-audio", false)?)
} else {
None
};
let mic = if publisher && tracks.microphone {
Some(add_sender(&pipeline, &rtc, 98, "microphone", false)?)
} else {
None
};
pipeline.set_state(gst::State::Playing).map_err(error)?;
Ok(Self {
pipeline,
rtc,
video,
app,
mic,
events,
tx,
remote,
remote_set: false,
started: std::time::Instant::now(),
pending_ice: Vec::new(),
app_time: Mutex::new(None),
mic_time: Mutex::new(None),
})
}
/// Create an SDP offer only after the gateway authorizes a viewer subscription.
pub fn offer(&self) {
create_description(&self.rtc, true, self.signal_sender());
}
fn signal_sender(&self) -> mpsc::SyncSender<Signal> {
self.tx.clone()
}
pub fn apply_offer(&mut self, sdp: &str) -> Result<(), String> {
self.set_description(sdp, true)?;
create_description(&self.rtc, false, self.signal_sender());
Ok(())
}
pub fn apply_answer(&mut self, sdp: &str) -> Result<(), String> {
self.set_description(sdp, false)
}
fn set_description(&mut self, text: &str, offer: bool) -> Result<(), String> {
if text.len() > 65_536 {
return Err("SDP exceeds limit".into());
}
let sdp = gstreamer_sdp::SDPMessage::parse_buffer(text.as_bytes()).map_err(error)?;
let description = rtc::WebRTCSessionDescription::new(
if offer {
rtc::WebRTCSDPType::Offer
} else {
rtc::WebRTCSDPType::Answer
},
sdp,
);
let promise = gst::Promise::new();
self.rtc
.emit_by_name::<()>("set-remote-description", &[&description, &promise]);
if promise.wait() != gst::PromiseResult::Replied {
return Err("Remote SDP was rejected".into());
}
if let Some(reply) = promise.get_reply()
&& reply.has_field("error")
{
return Err("Remote SDP was rejected".into());
}
self.remote_set = true;
for (mline, candidate) in self.pending_ice.drain(..) {
self.rtc
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
}
Ok(())
}
pub fn add_ice(&mut self, mline: u32, candidate: &str) -> Result<(), String> {
if mline > 2 || candidate.len() > 2048 || self.pending_ice.len() >= 64 {
return Err("ICE candidate exceeds limit".into());
}
if self.remote_set {
self.rtc
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
} else {
self.pending_ice.push((mline, candidate.into()));
}
Ok(())
}
pub fn poll(&self) -> Result<Option<Signal>, String> {
if let Some(bus) = self.pipeline.bus() {
while let Some(message) = bus.pop() {
match message.view() {
gst::MessageView::Error(e) => return Err(format!("WebRTC: {}", e.error())),
gst::MessageView::Eos(..) => return Err("WebRTC stream closed".into()),
_ => {}
}
}
}
let state = self
.rtc
.property::<rtc::WebRTCPeerConnectionState>("connection-state");
if self.started.elapsed() > std::time::Duration::from_secs(20)
&& state != rtc::WebRTCPeerConnectionState::Connected
{
return Err("WebRTC connection timed out".into());
}
if matches!(
state,
rtc::WebRTCPeerConnectionState::Failed | rtc::WebRTCPeerConnectionState::Closed
) {
return Err("WebRTC peer disconnected".into());
}
Ok(self.events.try_recv().ok())
}
pub fn connected(&self) -> bool {
self.rtc
.property::<rtc::WebRTCPeerConnectionState>("connection-state")
== rtc::WebRTCPeerConnectionState::Connected
}
pub fn push_video(&self, frame: &VideoFrame) -> Result<(), String> {
frame.validate()?;
let source = self.video.as_ref().ok_or("Peer does not publish video")?;
source.set_caps(Some(
&gst::Caps::builder("video/x-raw")
.field("format", "RGBA")
.field("width", frame.width as i32)
.field("height", frame.height as i32)
.field("framerate", gst::Fraction::new(30, 1))
.build(),
));
source
.push_buffer(gst::Buffer::from_mut_slice(frame.rgba.clone()))
.map_err(error)?;
Ok(())
}
pub fn push_audio(&self, track: AudioTrack, frame: &AudioFrame) -> Result<(), String> {
frame.validate()?;
let source = match track {
AudioTrack::Application => &self.app,
AudioTrack::Microphone => &self.mic,
}
.as_ref()
.ok_or("Audio track was not explicitly enabled")?;
source.set_caps(Some(
&gst::Caps::builder("audio/x-raw")
.field("format", "F32LE")
.field("layout", "interleaved")
.field("rate", frame.rate as i32)
.field("channels", i32::from(frame.channels))
.build(),
));
let bytes: Vec<u8> = frame
.samples
.iter()
.flat_map(|sample| sample.to_le_bytes())
.collect();
let clock = match track {
AudioTrack::Application => &self.app_time,
AudioTrack::Microphone => &self.mic_time,
};
let mut next = clock
.lock()
.map_err(|_| "Audio timestamp clock unavailable")?;
let pts = next.unwrap_or_else(|| {
self.pipeline
.current_running_time()
.map(|time| time.nseconds())
.unwrap_or(0)
});
let duration = (frame.samples.len() as u64 / u64::from(frame.channels)) * 1_000_000_000
/ u64::from(frame.rate);
let mut buffer = gst::Buffer::from_mut_slice(bytes);
if let Some(buffer) = buffer.get_mut() {
buffer.set_pts(gst::ClockTime::from_nseconds(pts));
buffer.set_duration(gst::ClockTime::from_nseconds(duration));
}
*next = Some(pts.saturating_add(duration));
source.push_buffer(buffer).map_err(error)?;
Ok(())
}
}
impl Drop for Peer {
fn drop(&mut self) {
let _ = self.pipeline.set_state(gst::State::Null);
}
}
fn add_sender(
pipeline: &gst::Pipeline,
rtc: &gst::Element,
payload: i32,
msid: &str,
video: bool,
) -> Result<gstreamer_app::AppSrc, String> {
let description = if video {
format!(
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=8388608 leaky-type=downstream ! queue max-size-buffers=2 leaky=downstream ! videoconvert ! vp8enc deadline=1 cpu-used=8 target-bitrate=1500000 keyframe-max-dist=60 ! rtpvp8pay pt={payload} ! capsfilter caps=\"application/x-rtp,media=video,encoding-name=VP8,payload={payload},clock-rate=90000\""
)
} else {
format!(
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=384000 leaky-type=downstream ! queue max-size-buffers=8 leaky=downstream ! audioconvert ! audioresample ! opusenc bitrate=64000 ! rtpopuspay pt={payload} ! capsfilter caps=\"application/x-rtp,media=audio,encoding-name=OPUS,payload={payload},clock-rate=48000\""
)
};
let bin = gst::parse::bin_from_description(&description, true).map_err(error)?;
let source = bin
.by_name("source")
.ok_or("Missing media appsrc")?
.downcast::<gstreamer_app::AppSrc>()
.map_err(|_| "Wrong media source type")?;
pipeline.add(&bin).map_err(error)?;
let src = bin.static_pad("src").ok_or("Missing RTP source pad")?;
let sink = rtc
.request_pad_simple("sink_%u")
.ok_or("WebRTC refused a media track")?;
sink.set_property("msid", msid);
let transceiver = sink.property::<rtc::WebRTCRTPTransceiver>("transceiver");
transceiver.set_property("direction", rtc::WebRTCRTPTransceiverDirection::Sendonly);
src.link(&sink).map_err(error)?;
Ok(source)
}
fn create_description(webrtc: &gst::Element, offer: bool, tx: mpsc::SyncSender<Signal>) {
let weak = webrtc.downgrade();
let promise = gst::Promise::with_change_func(move |reply| {
let result = (|| {
let reply = reply
.map_err(|e| format!("SDP promise failed: {e:?}"))?
.ok_or("No SDP reply")?;
let description = reply
.get::<rtc::WebRTCSessionDescription>(if offer { "offer" } else { "answer" })
.map_err(error)?;
let webrtc = weak.upgrade().ok_or("WebRTC peer closed")?;
webrtc.emit_by_name::<()>(
"set-local-description",
&[&description, &None::<gst::Promise>],
);
let text = description.sdp().as_text().map_err(error)?;
Ok::<_, String>(if offer {
Signal::Offer(text)
} else {
Signal::Answer(text)
})
})();
let _ = tx.try_send(result.unwrap_or_else(Signal::Error));
});
webrtc.emit_by_name::<()>(
if offer {
"create-offer"
} else {
"create-answer"
},
&[&None::<gst::Structure>, &promise],
);
}
fn receive(
pipeline: &gst::Pipeline,
pad: &gst::Pad,
feed: &RemoteFeed,
tracks: Tracks,
) -> Result<(), String> {
let caps = pad.current_caps().ok_or("Incoming track has no caps")?;
let payload = caps
.structure(0)
.ok_or("Incoming track has no format")?
.get::<i32>("payload")
.map_err(error)?;
let video = payload == 96;
if !(video || payload == 97 && tracks.application_audio || payload == 98 && tracks.microphone) {
return Err("Unannounced remote track rejected".into());
}
let description = if video {
"queue max-size-buffers=2 leaky=downstream ! rtpvp8depay ! vp8dec ! videoconvert ! video/x-raw,format=RGBA,width=[1,1920],height=[1,1080] ! appsink name=sink sync=false max-buffers=1 drop=true"
} else {
"queue max-size-buffers=8 leaky=downstream ! rtpopusdepay ! opusdec ! audioconvert ! audioresample ! audio/x-raw,format=F32LE,layout=interleaved,rate=48000,channels=2 ! appsink name=sink sync=false max-buffers=4 drop=true"
};
let bin = gst::parse::bin_from_description(description, true).map_err(error)?;
let sink = bin
.by_name("sink")
.ok_or("Remote sink missing")?
.downcast::<gstreamer_app::AppSink>()
.map_err(|_| "Wrong remote sink")?;
let feed = feed.clone();
sink.set_callbacks(
gstreamer_app::AppSinkCallbacks::builder()
.new_sample(move |sink| {
let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
if video {
let frame =
super::capture::copy_video(&sample).map_err(|_| gst::FlowError::Error)?;
*feed.video.lock().map_err(|_| gst::FlowError::Error)? = Some(Arc::new(frame));
feed.video_frames.fetch_add(1, Ordering::Relaxed);
} else {
let map = sample
.buffer()
.ok_or(gst::FlowError::Error)?
.map_readable()
.map_err(|_| gst::FlowError::Error)?;
if map.len() > 384000 {
return Err(gst::FlowError::Error);
}
let samples: Vec<f32> = map
.as_slice()
.as_chunks::<4>()
.0
.iter()
.map(|bytes| f32::from_le_bytes(*bytes))
.collect();
let count = samples.len() as u64;
let frame = AudioFrame {
samples,
rate: 48000,
channels: 2,
};
frame.validate().map_err(|_| gst::FlowError::Error)?;
let (queue, received) = if payload == 97 {
(&feed.app, &feed.application_samples)
} else {
(&feed.mic, &feed.microphone_samples)
};
let mut queue = queue.lock().map_err(|_| gst::FlowError::Error)?;
if queue.len() == 8 {
queue.pop_front();
}
queue.push_back(frame);
received.fetch_add(count, Ordering::Relaxed);
}
Ok(gst::FlowSuccess::Ok)
})
.build(),
);
pipeline.add(&bin).map_err(error)?;
bin.sync_state_with_parent().map_err(error)?;
pad.link(&bin.static_pad("sink").ok_or("Missing remote RTP sink")?)
.map_err(error)?;
Ok(())
}
fn error(error: impl std::fmt::Display) -> String {
error.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{Duration, Instant};
fn forward(from: &Peer, to: &mut Peer) {
while let Some(signal) = from.poll().expect("healthy peer") {
match signal {
Signal::Offer(sdp) => to.apply_offer(&sdp).expect("offer accepted"),
Signal::Answer(sdp) => to.apply_answer(&sdp).expect("answer accepted"),
Signal::Ice { mline, candidate } => {
to.add_ice(mline, &candidate).expect("candidate accepted")
}
Signal::Error(error) => panic!("signalling failed: {error}"),
}
}
}
#[test]
fn two_peers_deliver_video_and_separate_application_and_microphone_tracks() {
let tracks = Tracks {
application_audio: true,
microphone: true,
};
let mut publisher = Peer::publisher(tracks).expect("publisher");
let mut viewer = Peer::viewer(tracks).expect("viewer");
let frame = VideoFrame {
width: 64,
height: 48,
rgba: [200, 50, 70, 255].repeat(64 * 48),
timestamp_ns: None,
};
publisher.push_video(&frame).expect("fixture video");
publisher.offer();
let start = Instant::now();
let mut index = 0usize;
let mut app_pcm = Vec::new();
let mut mic_pcm = Vec::new();
while start.elapsed() < Duration::from_secs(12) {
forward(&publisher, &mut viewer);
forward(&viewer, &mut publisher);
publisher.push_video(&frame).expect("video push");
for (track, frequency) in [
(AudioTrack::Application, 440.0f32),
(AudioTrack::Microphone, 880.0f32),
] {
let samples = (index..index + 960)
.flat_map(|sample| {
let value = (sample as f32 * frequency * std::f32::consts::TAU / 48_000.0)
.sin()
* 0.3;
[value, value]
})
.collect();
publisher
.push_audio(
track,
&AudioFrame {
samples,
rate: 48_000,
channels: 2,
},
)
.expect("independent audio push");
}
index += 960;
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Application) {
app_pcm.extend(frame.samples.into_iter().step_by(2));
}
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Microphone) {
mic_pcm.extend(frame.samples.into_iter().step_by(2));
}
if viewer.remote.video_frames.load(Ordering::Relaxed) >= 3
&& app_pcm.len() > 24_000
&& mic_pcm.len() > 24_000
{
break;
}
std::thread::sleep(Duration::from_millis(20));
}
assert!(
publisher.connected() && viewer.connected(),
"DTLS/ICE connection established"
);
let received = viewer
.remote
.take_video()
.expect("actual decoded remote video");
assert_eq!((received.width, received.height), (64, 48));
assert!((i32::from(received.rgba[0]) - 200).abs() < 20);
assert!(
app_pcm.len() > 24_000 && mic_pcm.len() > 24_000,
"both distinct Opus tracks delivered"
);
let frequency = |pcm: &[f32]| {
pcm.windows(2)
.filter(|pair| pair[0] < 0.0 && pair[1] >= 0.0)
.count() as f32
* 48_000.0
/ pcm.len() as f32
};
let app_frequency = frequency(&app_pcm);
let mic_frequency = frequency(&mic_pcm);
assert!(
(app_frequency - 440.0).abs() < 60.0,
"app frequency {app_frequency}"
);
assert!(
(mic_frequency - 880.0).abs() < 100.0,
"microphone frequency {mic_frequency}"
);
assert!(
publisher
.push_audio(
AudioTrack::Application,
&AudioFrame {
samples: vec![f32::NAN],
rate: 48_000,
channels: 1
}
)
.is_err()
);
}
#[test]
fn unrequested_audio_track_cannot_be_enabled_by_push() {
let publisher = Peer::publisher(Tracks::default()).expect("publisher");
let pcm = AudioFrame {
samples: vec![0.0; 960],
rate: 48_000,
channels: 1,
};
assert!(publisher.push_audio(AudioTrack::Microphone, &pcm).is_err());
assert!(publisher.push_audio(AudioTrack::Application, &pcm).is_err());
}
}