feat(client): protocol v2 wire layer, screen sharing, channel webhooks and secure voice
Verified against the real gateway via tests/gateway_interop.rs.
This commit is contained in:
parent
8460fb28fd
commit
3d7dd5413b
65 changed files with 6048 additions and 1156 deletions
646
src/media/peer.rs
Normal file
646
src/media/peer.rs
Normal file
|
|
@ -0,0 +1,646 @@
|
|||
//! Established ICE/DTLS-SRTP transport via GStreamer webrtcbin, with independent tracks.
|
||||
use super::{AudioFrame, VideoFrame};
|
||||
use gst::prelude::*;
|
||||
use gstreamer as gst;
|
||||
use gstreamer_webrtc as rtc;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
mpsc,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct Tracks {
|
||||
pub application_audio: bool,
|
||||
pub microphone: bool,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum Signal {
|
||||
Offer(String),
|
||||
Answer(String),
|
||||
Ice { mline: u32, candidate: String },
|
||||
Error(String),
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum AudioTrack {
|
||||
Application,
|
||||
Microphone,
|
||||
}
|
||||
#[derive(Clone, Default)]
|
||||
pub struct RemoteFeed {
|
||||
video: Arc<Mutex<Option<Arc<VideoFrame>>>>,
|
||||
app: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
|
||||
mic: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
|
||||
pub video_frames: Arc<AtomicU64>,
|
||||
pub application_samples: Arc<AtomicU64>,
|
||||
pub microphone_samples: Arc<AtomicU64>,
|
||||
}
|
||||
impl RemoteFeed {
|
||||
pub fn take_video(&self) -> Option<Arc<VideoFrame>> {
|
||||
self.video.lock().ok()?.take()
|
||||
}
|
||||
pub fn take_audio(&self, track: AudioTrack) -> Option<AudioFrame> {
|
||||
match track {
|
||||
AudioTrack::Application => self.app.lock().ok()?.pop_front(),
|
||||
AudioTrack::Microphone => self.mic.lock().ok()?.pop_front(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit administrator-configured ICE services. Empty means host candidates only.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct IceServers {
|
||||
pub stun: Option<String>,
|
||||
pub turn: Option<String>,
|
||||
}
|
||||
impl IceServers {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.stun.as_ref().is_some_and(|uri| {
|
||||
!uri.starts_with("stun://")
|
||||
|| uri.len() > 2048
|
||||
|| uri.len() <= 7
|
||||
|| uri.chars().any(char::is_whitespace)
|
||||
}) || self.turn.as_ref().is_some_and(|uri| {
|
||||
!(uri.starts_with("turn://") || uri.starts_with("turns://"))
|
||||
|| uri.len() > 2048
|
||||
|| uri.len() <= 7
|
||||
|| uri.chars().any(char::is_whitespace)
|
||||
}) {
|
||||
return Err("Invalid explicitly configured ICE server URI".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// One directed publisher/viewer connection. The authenticated gateway routes its signals.
|
||||
pub struct Peer {
|
||||
pipeline: gst::Pipeline,
|
||||
rtc: gst::Element,
|
||||
video: Option<gstreamer_app::AppSrc>,
|
||||
app: Option<gstreamer_app::AppSrc>,
|
||||
mic: Option<gstreamer_app::AppSrc>,
|
||||
events: mpsc::Receiver<Signal>,
|
||||
tx: mpsc::SyncSender<Signal>,
|
||||
pub remote: RemoteFeed,
|
||||
remote_set: bool,
|
||||
started: std::time::Instant,
|
||||
pending_ice: Vec<(u32, String)>,
|
||||
app_time: Mutex<Option<u64>>,
|
||||
mic_time: Mutex<Option<u64>>,
|
||||
}
|
||||
|
||||
pub fn check_backend() -> Result<(), String> {
|
||||
gst::init().map_err(error)?;
|
||||
for plugin in [
|
||||
"webrtcbin",
|
||||
"nicesrc",
|
||||
"nicesink",
|
||||
"dtlssrtpenc",
|
||||
"dtlssrtpdec",
|
||||
"vp8enc",
|
||||
"vp8dec",
|
||||
"rtpvp8pay",
|
||||
"rtpvp8depay",
|
||||
"opusenc",
|
||||
"opusdec",
|
||||
"rtpopuspay",
|
||||
"rtpopusdepay",
|
||||
] {
|
||||
if gst::ElementFactory::find(plugin).is_none() {
|
||||
return Err(format!("Missing WebRTC plugin: {plugin}"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
impl Peer {
|
||||
pub fn publisher(tracks: Tracks) -> Result<Self, String> {
|
||||
Self::new(true, tracks, &IceServers::default())
|
||||
}
|
||||
pub fn viewer(tracks: Tracks) -> Result<Self, String> {
|
||||
Self::new(false, tracks, &IceServers::default())
|
||||
}
|
||||
pub fn publisher_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
|
||||
Self::new(true, tracks, ice)
|
||||
}
|
||||
pub fn viewer_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
|
||||
Self::new(false, tracks, ice)
|
||||
}
|
||||
fn new(publisher: bool, tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
|
||||
ice.validate()?;
|
||||
check_backend()?;
|
||||
let pipeline = gst::Pipeline::new();
|
||||
let rtc = gst::ElementFactory::make("webrtcbin")
|
||||
.name("transport")
|
||||
.build()
|
||||
.map_err(error)?;
|
||||
rtc.set_property_from_str("bundle-policy", "max-bundle");
|
||||
rtc.set_property("latency", 100u32);
|
||||
if let Some(uri) = &ice.stun {
|
||||
rtc.set_property("stun-server", uri);
|
||||
}
|
||||
if let Some(uri) = &ice.turn {
|
||||
rtc.set_property("turn-server", uri);
|
||||
}
|
||||
pipeline.add(&rtc).map_err(error)?;
|
||||
let (tx, events) = mpsc::sync_channel(64);
|
||||
let ice_tx = tx.clone();
|
||||
rtc.connect("on-ice-candidate", false, move |values| {
|
||||
if let (Ok(mline), Ok(candidate)) = (values[1].get::<u32>(), values[2].get::<String>())
|
||||
{
|
||||
let _ = ice_tx.try_send(Signal::Ice { mline, candidate });
|
||||
}
|
||||
None
|
||||
});
|
||||
let remote = RemoteFeed::default();
|
||||
if !publisher {
|
||||
let weak_pipeline = pipeline.downgrade();
|
||||
let feed = remote.clone();
|
||||
let receive_tx = tx.clone();
|
||||
rtc.connect_pad_added(move |_, pad| {
|
||||
if pad.direction() != gst::PadDirection::Src {
|
||||
return;
|
||||
}
|
||||
if let Some(pipeline) = weak_pipeline.upgrade()
|
||||
&& let Err(error) = receive(&pipeline, pad, &feed, tracks)
|
||||
{
|
||||
let _ = receive_tx.try_send(Signal::Error(error));
|
||||
}
|
||||
});
|
||||
}
|
||||
let video = if publisher {
|
||||
Some(add_sender(&pipeline, &rtc, 96, "screen-video", true)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let app = if publisher && tracks.application_audio {
|
||||
Some(add_sender(&pipeline, &rtc, 97, "application-audio", false)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mic = if publisher && tracks.microphone {
|
||||
Some(add_sender(&pipeline, &rtc, 98, "microphone", false)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
pipeline.set_state(gst::State::Playing).map_err(error)?;
|
||||
Ok(Self {
|
||||
pipeline,
|
||||
rtc,
|
||||
video,
|
||||
app,
|
||||
mic,
|
||||
events,
|
||||
tx,
|
||||
remote,
|
||||
remote_set: false,
|
||||
started: std::time::Instant::now(),
|
||||
pending_ice: Vec::new(),
|
||||
app_time: Mutex::new(None),
|
||||
mic_time: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
/// Create an SDP offer only after the gateway authorizes a viewer subscription.
|
||||
pub fn offer(&self) {
|
||||
create_description(&self.rtc, true, self.signal_sender());
|
||||
}
|
||||
fn signal_sender(&self) -> mpsc::SyncSender<Signal> {
|
||||
self.tx.clone()
|
||||
}
|
||||
pub fn apply_offer(&mut self, sdp: &str) -> Result<(), String> {
|
||||
self.set_description(sdp, true)?;
|
||||
create_description(&self.rtc, false, self.signal_sender());
|
||||
Ok(())
|
||||
}
|
||||
pub fn apply_answer(&mut self, sdp: &str) -> Result<(), String> {
|
||||
self.set_description(sdp, false)
|
||||
}
|
||||
fn set_description(&mut self, text: &str, offer: bool) -> Result<(), String> {
|
||||
if text.len() > 65_536 {
|
||||
return Err("SDP exceeds limit".into());
|
||||
}
|
||||
let sdp = gstreamer_sdp::SDPMessage::parse_buffer(text.as_bytes()).map_err(error)?;
|
||||
let description = rtc::WebRTCSessionDescription::new(
|
||||
if offer {
|
||||
rtc::WebRTCSDPType::Offer
|
||||
} else {
|
||||
rtc::WebRTCSDPType::Answer
|
||||
},
|
||||
sdp,
|
||||
);
|
||||
let promise = gst::Promise::new();
|
||||
self.rtc
|
||||
.emit_by_name::<()>("set-remote-description", &[&description, &promise]);
|
||||
if promise.wait() != gst::PromiseResult::Replied {
|
||||
return Err("Remote SDP was rejected".into());
|
||||
}
|
||||
if let Some(reply) = promise.get_reply()
|
||||
&& reply.has_field("error")
|
||||
{
|
||||
return Err("Remote SDP was rejected".into());
|
||||
}
|
||||
self.remote_set = true;
|
||||
for (mline, candidate) in self.pending_ice.drain(..) {
|
||||
self.rtc
|
||||
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn add_ice(&mut self, mline: u32, candidate: &str) -> Result<(), String> {
|
||||
if mline > 2 || candidate.len() > 2048 || self.pending_ice.len() >= 64 {
|
||||
return Err("ICE candidate exceeds limit".into());
|
||||
}
|
||||
if self.remote_set {
|
||||
self.rtc
|
||||
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
|
||||
} else {
|
||||
self.pending_ice.push((mline, candidate.into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn poll(&self) -> Result<Option<Signal>, String> {
|
||||
if let Some(bus) = self.pipeline.bus() {
|
||||
while let Some(message) = bus.pop() {
|
||||
match message.view() {
|
||||
gst::MessageView::Error(e) => return Err(format!("WebRTC: {}", e.error())),
|
||||
gst::MessageView::Eos(..) => return Err("WebRTC stream closed".into()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let state = self
|
||||
.rtc
|
||||
.property::<rtc::WebRTCPeerConnectionState>("connection-state");
|
||||
if self.started.elapsed() > std::time::Duration::from_secs(20)
|
||||
&& state != rtc::WebRTCPeerConnectionState::Connected
|
||||
{
|
||||
return Err("WebRTC connection timed out".into());
|
||||
}
|
||||
if matches!(
|
||||
state,
|
||||
rtc::WebRTCPeerConnectionState::Failed | rtc::WebRTCPeerConnectionState::Closed
|
||||
) {
|
||||
return Err("WebRTC peer disconnected".into());
|
||||
}
|
||||
Ok(self.events.try_recv().ok())
|
||||
}
|
||||
pub fn connected(&self) -> bool {
|
||||
self.rtc
|
||||
.property::<rtc::WebRTCPeerConnectionState>("connection-state")
|
||||
== rtc::WebRTCPeerConnectionState::Connected
|
||||
}
|
||||
pub fn push_video(&self, frame: &VideoFrame) -> Result<(), String> {
|
||||
frame.validate()?;
|
||||
let source = self.video.as_ref().ok_or("Peer does not publish video")?;
|
||||
source.set_caps(Some(
|
||||
&gst::Caps::builder("video/x-raw")
|
||||
.field("format", "RGBA")
|
||||
.field("width", frame.width as i32)
|
||||
.field("height", frame.height as i32)
|
||||
.field("framerate", gst::Fraction::new(30, 1))
|
||||
.build(),
|
||||
));
|
||||
source
|
||||
.push_buffer(gst::Buffer::from_mut_slice(frame.rgba.clone()))
|
||||
.map_err(error)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn push_audio(&self, track: AudioTrack, frame: &AudioFrame) -> Result<(), String> {
|
||||
frame.validate()?;
|
||||
let source = match track {
|
||||
AudioTrack::Application => &self.app,
|
||||
AudioTrack::Microphone => &self.mic,
|
||||
}
|
||||
.as_ref()
|
||||
.ok_or("Audio track was not explicitly enabled")?;
|
||||
source.set_caps(Some(
|
||||
&gst::Caps::builder("audio/x-raw")
|
||||
.field("format", "F32LE")
|
||||
.field("layout", "interleaved")
|
||||
.field("rate", frame.rate as i32)
|
||||
.field("channels", i32::from(frame.channels))
|
||||
.build(),
|
||||
));
|
||||
let bytes: Vec<u8> = frame
|
||||
.samples
|
||||
.iter()
|
||||
.flat_map(|sample| sample.to_le_bytes())
|
||||
.collect();
|
||||
let clock = match track {
|
||||
AudioTrack::Application => &self.app_time,
|
||||
AudioTrack::Microphone => &self.mic_time,
|
||||
};
|
||||
let mut next = clock
|
||||
.lock()
|
||||
.map_err(|_| "Audio timestamp clock unavailable")?;
|
||||
let pts = next.unwrap_or_else(|| {
|
||||
self.pipeline
|
||||
.current_running_time()
|
||||
.map(|time| time.nseconds())
|
||||
.unwrap_or(0)
|
||||
});
|
||||
let duration = (frame.samples.len() as u64 / u64::from(frame.channels)) * 1_000_000_000
|
||||
/ u64::from(frame.rate);
|
||||
let mut buffer = gst::Buffer::from_mut_slice(bytes);
|
||||
if let Some(buffer) = buffer.get_mut() {
|
||||
buffer.set_pts(gst::ClockTime::from_nseconds(pts));
|
||||
buffer.set_duration(gst::ClockTime::from_nseconds(duration));
|
||||
}
|
||||
*next = Some(pts.saturating_add(duration));
|
||||
source.push_buffer(buffer).map_err(error)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
impl Drop for Peer {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.pipeline.set_state(gst::State::Null);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_sender(
|
||||
pipeline: &gst::Pipeline,
|
||||
rtc: &gst::Element,
|
||||
payload: i32,
|
||||
msid: &str,
|
||||
video: bool,
|
||||
) -> Result<gstreamer_app::AppSrc, String> {
|
||||
let description = if video {
|
||||
format!(
|
||||
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=8388608 leaky-type=downstream ! queue max-size-buffers=2 leaky=downstream ! videoconvert ! vp8enc deadline=1 cpu-used=8 target-bitrate=1500000 keyframe-max-dist=60 ! rtpvp8pay pt={payload} ! capsfilter caps=\"application/x-rtp,media=video,encoding-name=VP8,payload={payload},clock-rate=90000\""
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=384000 leaky-type=downstream ! queue max-size-buffers=8 leaky=downstream ! audioconvert ! audioresample ! opusenc bitrate=64000 ! rtpopuspay pt={payload} ! capsfilter caps=\"application/x-rtp,media=audio,encoding-name=OPUS,payload={payload},clock-rate=48000\""
|
||||
)
|
||||
};
|
||||
let bin = gst::parse::bin_from_description(&description, true).map_err(error)?;
|
||||
let source = bin
|
||||
.by_name("source")
|
||||
.ok_or("Missing media appsrc")?
|
||||
.downcast::<gstreamer_app::AppSrc>()
|
||||
.map_err(|_| "Wrong media source type")?;
|
||||
pipeline.add(&bin).map_err(error)?;
|
||||
let src = bin.static_pad("src").ok_or("Missing RTP source pad")?;
|
||||
let sink = rtc
|
||||
.request_pad_simple("sink_%u")
|
||||
.ok_or("WebRTC refused a media track")?;
|
||||
sink.set_property("msid", msid);
|
||||
let transceiver = sink.property::<rtc::WebRTCRTPTransceiver>("transceiver");
|
||||
transceiver.set_property("direction", rtc::WebRTCRTPTransceiverDirection::Sendonly);
|
||||
src.link(&sink).map_err(error)?;
|
||||
Ok(source)
|
||||
}
|
||||
|
||||
fn create_description(webrtc: &gst::Element, offer: bool, tx: mpsc::SyncSender<Signal>) {
|
||||
let weak = webrtc.downgrade();
|
||||
let promise = gst::Promise::with_change_func(move |reply| {
|
||||
let result = (|| {
|
||||
let reply = reply
|
||||
.map_err(|e| format!("SDP promise failed: {e:?}"))?
|
||||
.ok_or("No SDP reply")?;
|
||||
let description = reply
|
||||
.get::<rtc::WebRTCSessionDescription>(if offer { "offer" } else { "answer" })
|
||||
.map_err(error)?;
|
||||
let webrtc = weak.upgrade().ok_or("WebRTC peer closed")?;
|
||||
webrtc.emit_by_name::<()>(
|
||||
"set-local-description",
|
||||
&[&description, &None::<gst::Promise>],
|
||||
);
|
||||
let text = description.sdp().as_text().map_err(error)?;
|
||||
Ok::<_, String>(if offer {
|
||||
Signal::Offer(text)
|
||||
} else {
|
||||
Signal::Answer(text)
|
||||
})
|
||||
})();
|
||||
let _ = tx.try_send(result.unwrap_or_else(Signal::Error));
|
||||
});
|
||||
webrtc.emit_by_name::<()>(
|
||||
if offer {
|
||||
"create-offer"
|
||||
} else {
|
||||
"create-answer"
|
||||
},
|
||||
&[&None::<gst::Structure>, &promise],
|
||||
);
|
||||
}
|
||||
|
||||
fn receive(
|
||||
pipeline: &gst::Pipeline,
|
||||
pad: &gst::Pad,
|
||||
feed: &RemoteFeed,
|
||||
tracks: Tracks,
|
||||
) -> Result<(), String> {
|
||||
let caps = pad.current_caps().ok_or("Incoming track has no caps")?;
|
||||
let payload = caps
|
||||
.structure(0)
|
||||
.ok_or("Incoming track has no format")?
|
||||
.get::<i32>("payload")
|
||||
.map_err(error)?;
|
||||
let video = payload == 96;
|
||||
if !(video || payload == 97 && tracks.application_audio || payload == 98 && tracks.microphone) {
|
||||
return Err("Unannounced remote track rejected".into());
|
||||
}
|
||||
let description = if video {
|
||||
"queue max-size-buffers=2 leaky=downstream ! rtpvp8depay ! vp8dec ! videoconvert ! video/x-raw,format=RGBA,width=[1,1920],height=[1,1080] ! appsink name=sink sync=false max-buffers=1 drop=true"
|
||||
} else {
|
||||
"queue max-size-buffers=8 leaky=downstream ! rtpopusdepay ! opusdec ! audioconvert ! audioresample ! audio/x-raw,format=F32LE,layout=interleaved,rate=48000,channels=2 ! appsink name=sink sync=false max-buffers=4 drop=true"
|
||||
};
|
||||
let bin = gst::parse::bin_from_description(description, true).map_err(error)?;
|
||||
let sink = bin
|
||||
.by_name("sink")
|
||||
.ok_or("Remote sink missing")?
|
||||
.downcast::<gstreamer_app::AppSink>()
|
||||
.map_err(|_| "Wrong remote sink")?;
|
||||
let feed = feed.clone();
|
||||
sink.set_callbacks(
|
||||
gstreamer_app::AppSinkCallbacks::builder()
|
||||
.new_sample(move |sink| {
|
||||
let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
|
||||
if video {
|
||||
let frame =
|
||||
super::capture::copy_video(&sample).map_err(|_| gst::FlowError::Error)?;
|
||||
*feed.video.lock().map_err(|_| gst::FlowError::Error)? = Some(Arc::new(frame));
|
||||
feed.video_frames.fetch_add(1, Ordering::Relaxed);
|
||||
} else {
|
||||
let map = sample
|
||||
.buffer()
|
||||
.ok_or(gst::FlowError::Error)?
|
||||
.map_readable()
|
||||
.map_err(|_| gst::FlowError::Error)?;
|
||||
if map.len() > 384000 {
|
||||
return Err(gst::FlowError::Error);
|
||||
}
|
||||
let samples: Vec<f32> = map
|
||||
.as_slice()
|
||||
.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|bytes| f32::from_le_bytes(*bytes))
|
||||
.collect();
|
||||
let count = samples.len() as u64;
|
||||
let frame = AudioFrame {
|
||||
samples,
|
||||
rate: 48000,
|
||||
channels: 2,
|
||||
};
|
||||
frame.validate().map_err(|_| gst::FlowError::Error)?;
|
||||
let (queue, received) = if payload == 97 {
|
||||
(&feed.app, &feed.application_samples)
|
||||
} else {
|
||||
(&feed.mic, &feed.microphone_samples)
|
||||
};
|
||||
let mut queue = queue.lock().map_err(|_| gst::FlowError::Error)?;
|
||||
if queue.len() == 8 {
|
||||
queue.pop_front();
|
||||
}
|
||||
queue.push_back(frame);
|
||||
received.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
Ok(gst::FlowSuccess::Ok)
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
pipeline.add(&bin).map_err(error)?;
|
||||
bin.sync_state_with_parent().map_err(error)?;
|
||||
pad.link(&bin.static_pad("sink").ok_or("Missing remote RTP sink")?)
|
||||
.map_err(error)?;
|
||||
Ok(())
|
||||
}
|
||||
fn error(error: impl std::fmt::Display) -> String {
|
||||
error.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::{Duration, Instant};
|
||||
fn forward(from: &Peer, to: &mut Peer) {
|
||||
while let Some(signal) = from.poll().expect("healthy peer") {
|
||||
match signal {
|
||||
Signal::Offer(sdp) => to.apply_offer(&sdp).expect("offer accepted"),
|
||||
Signal::Answer(sdp) => to.apply_answer(&sdp).expect("answer accepted"),
|
||||
Signal::Ice { mline, candidate } => {
|
||||
to.add_ice(mline, &candidate).expect("candidate accepted")
|
||||
}
|
||||
Signal::Error(error) => panic!("signalling failed: {error}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn two_peers_deliver_video_and_separate_application_and_microphone_tracks() {
|
||||
let tracks = Tracks {
|
||||
application_audio: true,
|
||||
microphone: true,
|
||||
};
|
||||
let mut publisher = Peer::publisher(tracks).expect("publisher");
|
||||
let mut viewer = Peer::viewer(tracks).expect("viewer");
|
||||
let frame = VideoFrame {
|
||||
width: 64,
|
||||
height: 48,
|
||||
rgba: [200, 50, 70, 255].repeat(64 * 48),
|
||||
timestamp_ns: None,
|
||||
};
|
||||
publisher.push_video(&frame).expect("fixture video");
|
||||
publisher.offer();
|
||||
let start = Instant::now();
|
||||
let mut index = 0usize;
|
||||
let mut app_pcm = Vec::new();
|
||||
let mut mic_pcm = Vec::new();
|
||||
while start.elapsed() < Duration::from_secs(12) {
|
||||
forward(&publisher, &mut viewer);
|
||||
forward(&viewer, &mut publisher);
|
||||
publisher.push_video(&frame).expect("video push");
|
||||
for (track, frequency) in [
|
||||
(AudioTrack::Application, 440.0f32),
|
||||
(AudioTrack::Microphone, 880.0f32),
|
||||
] {
|
||||
let samples = (index..index + 960)
|
||||
.flat_map(|sample| {
|
||||
let value = (sample as f32 * frequency * std::f32::consts::TAU / 48_000.0)
|
||||
.sin()
|
||||
* 0.3;
|
||||
[value, value]
|
||||
})
|
||||
.collect();
|
||||
publisher
|
||||
.push_audio(
|
||||
track,
|
||||
&AudioFrame {
|
||||
samples,
|
||||
rate: 48_000,
|
||||
channels: 2,
|
||||
},
|
||||
)
|
||||
.expect("independent audio push");
|
||||
}
|
||||
index += 960;
|
||||
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Application) {
|
||||
app_pcm.extend(frame.samples.into_iter().step_by(2));
|
||||
}
|
||||
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Microphone) {
|
||||
mic_pcm.extend(frame.samples.into_iter().step_by(2));
|
||||
}
|
||||
if viewer.remote.video_frames.load(Ordering::Relaxed) >= 3
|
||||
&& app_pcm.len() > 24_000
|
||||
&& mic_pcm.len() > 24_000
|
||||
{
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
assert!(
|
||||
publisher.connected() && viewer.connected(),
|
||||
"DTLS/ICE connection established"
|
||||
);
|
||||
let received = viewer
|
||||
.remote
|
||||
.take_video()
|
||||
.expect("actual decoded remote video");
|
||||
assert_eq!((received.width, received.height), (64, 48));
|
||||
assert!((i32::from(received.rgba[0]) - 200).abs() < 20);
|
||||
assert!(
|
||||
app_pcm.len() > 24_000 && mic_pcm.len() > 24_000,
|
||||
"both distinct Opus tracks delivered"
|
||||
);
|
||||
let frequency = |pcm: &[f32]| {
|
||||
pcm.windows(2)
|
||||
.filter(|pair| pair[0] < 0.0 && pair[1] >= 0.0)
|
||||
.count() as f32
|
||||
* 48_000.0
|
||||
/ pcm.len() as f32
|
||||
};
|
||||
let app_frequency = frequency(&app_pcm);
|
||||
let mic_frequency = frequency(&mic_pcm);
|
||||
assert!(
|
||||
(app_frequency - 440.0).abs() < 60.0,
|
||||
"app frequency {app_frequency}"
|
||||
);
|
||||
assert!(
|
||||
(mic_frequency - 880.0).abs() < 100.0,
|
||||
"microphone frequency {mic_frequency}"
|
||||
);
|
||||
assert!(
|
||||
publisher
|
||||
.push_audio(
|
||||
AudioTrack::Application,
|
||||
&AudioFrame {
|
||||
samples: vec![f32::NAN],
|
||||
rate: 48_000,
|
||||
channels: 1
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn unrequested_audio_track_cannot_be_enabled_by_push() {
|
||||
let publisher = Peer::publisher(Tracks::default()).expect("publisher");
|
||||
let pcm = AudioFrame {
|
||||
samples: vec![0.0; 960],
|
||||
rate: 48_000,
|
||||
channels: 1,
|
||||
};
|
||||
assert!(publisher.push_audio(AudioTrack::Microphone, &pcm).is_err());
|
||||
assert!(publisher.push_audio(AudioTrack::Application, &pcm).is_err());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue