feat(client): protocol v2 wire layer, screen sharing, channel webhooks and secure voice

Verified against the real gateway via tests/gateway_interop.rs.
This commit is contained in:
Loki 2026-10-24 16:51:15 +02:00
parent 8460fb28fd
commit 3d7dd5413b
Signed by: boba
GPG key ID: 253067914055423B
65 changed files with 6048 additions and 1156 deletions

575
src/media/broadcast.rs Normal file
View file

@ -0,0 +1,575 @@
//! UI-independent broadcast controller. User actions enqueue authenticated LNEx controls.
use super::{
AudioFrame, CaptureFeed, VideoFrame,
peer::{AudioTrack, IceServers, Peer, RemoteFeed, Signal, Tracks},
};
use crate::net::wire::{MediaControl, MediaEvent};
use std::{
collections::{HashMap, VecDeque},
sync::Arc,
};
use tokio::sync::broadcast;
const START: u32 = 1;
const STOP: u32 = 2;
const WATCH: u32 = 3;
const UNWATCH: u32 = 4;
const OFFER: u32 = 5;
const ANSWER: u32 = 6;
const ICE: u32 = 7;
#[derive(Clone, Debug)]
pub struct AvailableShare {
pub id: String,
pub publisher_session: String,
pub publisher_user: String,
pub application_audio: bool,
pub microphone: bool,
}
struct Publisher {
id: String,
tracks: Tracks,
acknowledged: bool,
started: std::time::Instant,
video: broadcast::Receiver<Arc<VideoFrame>>,
audio: broadcast::Receiver<Arc<AudioFrame>>,
latest: Option<Arc<VideoFrame>>,
peers: HashMap<String, Peer>,
}
struct Viewer {
share: AvailableShare,
peer: Option<Peer>,
pending_ice: Vec<(u32, String)>,
started: std::time::Instant,
}
#[derive(Default)]
pub struct BroadcastManager {
session: String,
channel: Option<String>,
available: HashMap<String, AvailableShare>,
publisher: Option<Publisher>,
viewers: HashMap<String, Viewer>,
outgoing: VecDeque<MediaControl>,
errors: VecDeque<String>,
ice: IceServers,
}
impl BroadcastManager {
pub fn configure_ice(&mut self, ice: IceServers) -> Result<(), String> {
ice.validate()?;
if self.publisher.is_some() || !self.viewers.is_empty() {
return Err("Stop broadcasts before changing ICE services".into());
}
self.ice = ice;
Ok(())
}
/// Called after authenticated session/voice membership changes; revokes local peers first.
pub fn set_scope(&mut self, session: &str, channel: Option<&str>) {
if self.session == session && self.channel.as_deref() == channel {
return;
}
self.stop();
self.outgoing.clear();
self.available.clear();
self.session = session.into();
self.channel = channel.map(str::to_owned);
if self.channel.is_some() {
self.enqueue(WATCH, "", "", "", 0, Tracks::default());
}
}
/// Explicit user publication only. A portal preview alone never calls this method.
pub fn publish(&mut self, feed: &CaptureFeed, tracks: Tracks) -> Result<String, String> {
if self.channel.is_none() || self.session.is_empty() {
return Err("Join a voice channel before broadcasting".into());
}
if self.publisher.is_some() {
return Err("A broadcast is already active".into());
}
if self.outgoing.len() >= 120 {
return Err(
"Media signalling congested; wait before starting another broadcast".into(),
);
}
super::peer::check_backend()?;
let id = uuid::Uuid::new_v4().to_string();
self.publisher = Some(Publisher {
id: id.clone(),
tracks,
acknowledged: false,
started: std::time::Instant::now(),
video: feed.subscribe_video(),
audio: feed.subscribe_audio(),
latest: None,
peers: HashMap::new(),
});
self.enqueue(START, &id, "", "", 0, tracks);
Ok(id)
}
pub fn stop_publishing(&mut self) {
if let Some(publisher) = self.publisher.take() {
self.enqueue(STOP, &publisher.id, "", "", 0, publisher.tracks);
}
}
/// Explicit receiver consent; no automatic incoming peer or speaker playback.
pub fn watch(&mut self, id: &str) -> Result<(), String> {
if self.viewers.contains_key(id) {
return Ok(());
}
if self.outgoing.len() >= 120 {
return Err(
"Media signalling congested; wait before starting another broadcast".into(),
);
}
if self.viewers.len() >= 4 {
return Err("At most four broadcasts may be viewed at once".into());
}
let share = self
.available
.get(id)
.ok_or("Broadcast is no longer available")?
.clone();
if share.publisher_session == self.session {
return Err("Cannot watch your own broadcast".into());
}
self.viewers.insert(
id.into(),
Viewer {
share,
peer: None,
pending_ice: Vec::new(),
started: std::time::Instant::now(),
},
);
self.enqueue(WATCH, id, "", "", 0, Tracks::default());
Ok(())
}
pub fn unwatch(&mut self, id: &str) {
if self.viewers.remove(id).is_some() {
self.enqueue(UNWATCH, id, "", "", 0, Tracks::default());
}
}
pub fn stop(&mut self) {
self.stop_publishing();
let ids: Vec<_> = self.viewers.keys().cloned().collect();
for id in ids {
self.unwatch(&id);
}
}
pub fn shares(&self) -> Vec<AvailableShare> {
let mut shares: Vec<_> = self.available.values().cloned().collect();
shares.sort_by(|a, b| a.id.cmp(&b.id));
shares
}
pub fn is_watching(&self, id: &str) -> bool {
self.viewers.contains_key(id)
}
pub fn publication_pending(&self) -> bool {
self.publisher.is_some()
}
pub fn publishing(&self) -> bool {
self.publisher
.as_ref()
.is_some_and(|publisher| publisher.acknowledged)
}
pub fn viewer_count(&self) -> usize {
self.publisher
.as_ref()
.map(|p| p.peers.values().filter(|peer| peer.connected()).count())
.unwrap_or(0)
}
pub fn remote_feed(&self, id: &str) -> Option<RemoteFeed> {
Some(self.viewers.get(id)?.peer.as_ref()?.remote.clone())
}
pub fn take_control(&mut self) -> Option<MediaControl> {
self.outgoing.pop_front()
}
pub fn take_error(&mut self) -> Option<String> {
self.errors.pop_front()
}
/// Existing voice capture supplies this only when explicitly enabled and unmuted.
pub fn push_microphone(&self, frame: &AudioFrame) -> Result<(), String> {
if let Some(publisher) = &self.publisher {
if !publisher.tracks.microphone {
return Ok(());
}
for peer in publisher.peers.values() {
peer.push_audio(AudioTrack::Microphone, frame)?;
}
}
Ok(())
}
pub fn on_event(&mut self, event: MediaEvent) -> Result<(), String> {
if Some(event.channel_id.as_str()) != self.channel.as_deref()
|| event.share_id.len() > 64
|| event.sender_session_id.len() > 64
{
return Err("Media event is outside the active voice scope".into());
}
match event.action {
START => {
if self.available.len() >= 128 && !self.available.contains_key(&event.share_id) {
return Err("Too many broadcast announcements".into());
}
let share = AvailableShare {
id: event.share_id.clone(),
publisher_session: event.sender_session_id.clone(),
publisher_user: event.publisher_user_id,
application_audio: event.application_audio,
microphone: event.microphone,
};
if let Some(publisher) = &mut self.publisher
&& publisher.id == event.share_id
&& event.sender_session_id == self.session
{
publisher.acknowledged = true;
}
self.available.insert(event.share_id, share);
}
STOP => {
if let Some(share) = self.available.get(&event.share_id)
&& share.publisher_session != event.sender_session_id
{
return Err("Stop sender is not the publisher".into());
}
self.available.remove(&event.share_id);
self.viewers.remove(&event.share_id);
if self
.publisher
.as_ref()
.is_some_and(|p| p.id == event.share_id)
{
self.publisher.take();
}
}
WATCH => {
if let Some(publisher) = &mut self.publisher
&& publisher.id == event.share_id
&& publisher.acknowledged
&& event.sender_session_id != self.session
{
if publisher.peers.len() >= 4 {
return Err("Viewer limit exceeded".into());
}
if !publisher.peers.contains_key(&event.sender_session_id) {
let peer = Peer::publisher_with_ice(publisher.tracks, &self.ice)?;
if let Some(frame) = &publisher.latest {
peer.push_video(frame)?;
}
peer.offer();
publisher.peers.insert(event.sender_session_id, peer);
}
}
}
UNWATCH => {
if let Some(publisher) = &mut self.publisher
&& publisher.id == event.share_id
{
publisher.peers.remove(&event.sender_session_id);
}
if self
.viewers
.get(&event.share_id)
.is_some_and(|v| v.share.publisher_session == event.sender_session_id)
{
self.viewers.remove(&event.share_id);
}
}
OFFER => {
let viewer = self
.viewers
.get_mut(&event.share_id)
.ok_or("Unrequested broadcast offer rejected")?;
if viewer.share.publisher_session != event.sender_session_id
|| viewer.peer.is_some()
{
return Err("Unexpected publisher or duplicate offer".into());
}
let mut peer = Peer::viewer_with_ice(
Tracks {
application_audio: viewer.share.application_audio,
microphone: viewer.share.microphone,
},
&self.ice,
)?;
peer.apply_offer(&event.body)?;
for (mline, candidate) in viewer.pending_ice.drain(..) {
peer.add_ice(mline, &candidate)?;
}
viewer.peer = Some(peer);
}
ANSWER => {
let publisher = self
.publisher
.as_mut()
.filter(|p| p.id == event.share_id)
.ok_or("No active publication for answer")?;
publisher
.peers
.get_mut(&event.sender_session_id)
.ok_or("Unsubscribed answer rejected")?
.apply_answer(&event.body)?;
}
ICE => {
if let Some(publisher) = self.publisher.as_mut().filter(|p| p.id == event.share_id)
{
publisher
.peers
.get_mut(&event.sender_session_id)
.ok_or("Unsubscribed ICE rejected")?
.add_ice(event.mline_index, &event.body)?;
} else {
let viewer = self
.viewers
.get_mut(&event.share_id)
.filter(|v| v.share.publisher_session == event.sender_session_id)
.ok_or("Unrequested ICE rejected")?;
if let Some(peer) = &mut viewer.peer {
peer.add_ice(event.mline_index, &event.body)?;
} else if viewer.pending_ice.len() < 64
&& event.body.len() <= 2048
&& event.mline_index <= 2
{
viewer.pending_ice.push((event.mline_index, event.body));
} else {
return Err("Too many early ICE candidates".into());
}
}
}
_ => return Err("Unknown media event".into()),
}
Ok(())
}
/// Call frequently on the owning UI thread; samples remain bounded and nonblocking.
pub fn tick(&mut self) {
if self.publisher.as_ref().is_some_and(|p| {
!p.acknowledged && p.started.elapsed() > std::time::Duration::from_secs(10)
}) {
self.stop_publishing();
self.errors
.push_back("Broadcast authorization timed out".into());
}
let expired: Vec<_> = self
.viewers
.iter()
.filter(|(_, v)| {
v.peer.is_none() && v.started.elapsed() > std::time::Duration::from_secs(20)
})
.map(|(id, _)| id.clone())
.collect();
for id in expired {
self.unwatch(&id);
if self.errors.len() < 8 {
self.errors
.push_back("Broadcast connection timed out".into());
}
}
let mut controls = Vec::new();
let mut failures = Vec::new();
if let Some(publisher) = &mut self.publisher {
let mut new_video = false;
loop {
match publisher.video.try_recv() {
Ok(frame) => {
publisher.latest = Some(frame);
new_video = true;
}
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
_ => break,
}
}
let mut audio = Vec::new();
loop {
match publisher.audio.try_recv() {
Ok(frame) => audio.push(frame),
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
_ => break,
}
}
for (viewer, peer) in &publisher.peers {
let result = (|| {
if new_video && let Some(frame) = &publisher.latest {
peer.push_video(frame)?;
}
for frame in &audio {
if publisher.tracks.application_audio {
peer.push_audio(AudioTrack::Application, frame)?;
}
}
while let Some(signal) = peer.poll()? {
controls.push((publisher.id.clone(), viewer.clone(), signal));
}
Ok::<_, String>(())
})();
if let Err(error) = result {
failures.push(error);
}
}
}
for (id, viewer) in &self.viewers {
if let Some(peer) = &viewer.peer {
loop {
match peer.poll() {
Ok(Some(signal)) => controls.push((
id.clone(),
viewer.share.publisher_session.clone(),
signal,
)),
Ok(None) => break,
Err(error) => {
failures.push(error);
break;
}
}
}
}
}
for (id, target, signal) in controls {
if !self
.publisher
.as_ref()
.is_some_and(|publisher| publisher.id == id)
&& !self.viewers.contains_key(&id)
{
continue;
}
match signal {
Signal::Offer(body) => {
self.enqueue(OFFER, &id, &target, &body, 0, Tracks::default())
}
Signal::Answer(body) => {
self.enqueue(ANSWER, &id, &target, &body, 0, Tracks::default())
}
Signal::Ice { mline, candidate } => {
self.enqueue(ICE, &id, &target, &candidate, mline, Tracks::default())
}
Signal::Error(error) => failures.push(error),
}
}
if !failures.is_empty() {
self.stop();
for error in failures {
if self.errors.len() < 8 {
self.errors.push_back(error);
}
}
}
}
fn enqueue(
&mut self,
action: u32,
id: &str,
target: &str,
body: &str,
mline_index: u32,
tracks: Tracks,
) {
if let Some(channel) = &self.channel {
if self.outgoing.len() >= 128 {
if self.errors.len() < 8 {
self.errors
.push_back("Media signalling queue overflow; broadcast stopped".into());
}
let mut cleanup: Vec<_> = self
.outgoing
.iter()
.filter(|control| matches!(control.action, STOP | UNWATCH))
.map(|control| (control.action, control.share_id.clone()))
.collect();
if let Some(publisher) = self.publisher.take() {
cleanup.push((STOP, publisher.id));
}
cleanup.extend(self.viewers.drain().map(|(id, _)| (UNWATCH, id)));
if matches!(action, STOP | UNWATCH)
&& !cleanup
.iter()
.any(|(kind, share)| *kind == action && share == id)
{
cleanup.push((action, id.into()));
}
cleanup.sort();
cleanup.dedup();
self.outgoing.clear();
for (action, share_id) in cleanup {
self.outgoing.push_back(MediaControl {
action,
channel_id: channel.clone(),
share_id,
..Default::default()
});
}
return;
}
self.outgoing.push_back(MediaControl {
action,
channel_id: channel.clone(),
share_id: id.into(),
target_session_id: target.into(),
body: body.into(),
mline_index,
application_audio: tracks.application_audio,
microphone: tracks.microphone,
});
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn signalling_overflow_discards_stale_offers_and_retains_stop() {
let mut manager = BroadcastManager::default();
manager.set_scope("publisher", Some("voice"));
let feed = CaptureFeed::default();
let id = manager
.publish(&feed, Tracks::default())
.expect("explicit publish");
manager.outgoing.clear(); // START already handed to authenticated networking.
for _ in 0..129 {
manager.enqueue(
ICE,
&id,
"viewer",
"candidate:fixture",
0,
Tracks::default(),
);
}
assert!(!manager.publication_pending());
let controls: Vec<_> = manager.outgoing.drain(..).collect();
assert_eq!(controls.len(), 1);
assert_eq!(controls[0].action, STOP);
assert_eq!(controls[0].share_id, id);
}
#[test]
fn overflow_preserves_queued_cleanup_for_previous_lifecycle() {
let mut manager = BroadcastManager::default();
manager.set_scope("publisher", Some("voice"));
let feed = CaptureFeed::default();
let old = manager
.publish(&feed, Tracks::default())
.expect("first publish");
manager.outgoing.clear();
manager.stop_publishing();
let new = manager
.publish(&feed, Tracks::default())
.expect("next publish");
for _ in 0..127 {
manager.enqueue(
ICE,
&new,
"viewer",
"candidate:fixture",
0,
Tracks::default(),
);
}
let stops: std::collections::HashSet<_> = manager
.outgoing
.iter()
.filter(|control| control.action == STOP)
.map(|control| control.share_id.clone())
.collect();
assert!(stops.contains(&old) && stops.contains(&new));
assert!(!manager.publication_pending());
}
}

View file

@ -12,14 +12,41 @@ use std::{
};
/// Latest owned preview frame and selected audio level; queues never grow.
#[derive(Clone, Default)]
#[derive(Clone)]
pub struct CaptureFeed {
video: Arc<Mutex<Option<VideoFrame>>>,
video: Arc<Mutex<Option<Arc<VideoFrame>>>>,
video_tx: Arc<tokio::sync::broadcast::Sender<Arc<VideoFrame>>>,
audio_tx: Arc<tokio::sync::broadcast::Sender<Arc<super::AudioFrame>>>,
audio: Arc<Mutex<f32>>,
received_audio: Arc<AtomicBool>,
}
impl Default for CaptureFeed {
fn default() -> Self {
Self {
video: Arc::default(),
audio: Arc::default(),
received_audio: Arc::default(),
video_tx: Arc::new(tokio::sync::broadcast::channel(2).0),
audio_tx: Arc::new(tokio::sync::broadcast::channel(8).0),
}
}
}
impl CaptureFeed {
pub fn take_frame(&self) -> Option<VideoFrame> {
#[cfg(test)]
pub(super) fn fixture_video(&self, frame: VideoFrame) {
let _ = self.video_tx.send(Arc::new(frame));
}
#[cfg(test)]
pub(super) fn fixture_audio(&self, frame: super::AudioFrame) {
let _ = self.audio_tx.send(Arc::new(frame));
}
pub fn subscribe_video(&self) -> tokio::sync::broadcast::Receiver<Arc<VideoFrame>> {
self.video_tx.subscribe()
}
pub fn subscribe_audio(&self) -> tokio::sync::broadcast::Receiver<Arc<super::AudioFrame>> {
self.audio_tx.subscribe()
}
pub fn take_frame(&self) -> Option<Arc<VideoFrame>> {
self.video.lock().ok()?.take()
}
pub fn audio_level(&self) -> f32 {
@ -87,6 +114,7 @@ impl CaptureSession {
source.set_property("path", portal.node.to_string());
}
strict_source(&source);
source.set_property("keepalive-time", 500i32);
let pipeline = video_pipeline(source, &session.feed, options)?;
session.pipelines.push(pipeline);
if let AudioChoice::Application(serial) | AudioChoice::System(serial) = options.audio {
@ -177,11 +205,13 @@ fn video_pipeline(
.sync(false)
.build();
let latest = feed.video.clone();
let tx = feed.video_tx.clone();
sink.set_callbacks(
gstreamer_app::AppSinkCallbacks::builder()
.new_sample(move |sink| {
let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
let frame = copy_video(&sample).map_err(|_| gst::FlowError::Error)?;
let frame = Arc::new(copy_video(&sample).map_err(|_| gst::FlowError::Error)?);
let _ = tx.send(frame.clone());
*latest.lock().map_err(|_| gst::FlowError::Error)? = Some(frame);
Ok(gst::FlowSuccess::Ok)
})
@ -214,7 +244,7 @@ fn video_pipeline(
Ok(pipeline)
}
fn copy_video(sample: &gst::Sample) -> Result<VideoFrame, String> {
pub(super) fn copy_video(sample: &gst::Sample) -> Result<VideoFrame, String> {
let caps = sample.caps().ok_or("Missing video caps")?;
let info = gstreamer_video::VideoInfo::from_caps(caps).map_err(error)?;
if info.format() != gstreamer_video::VideoFormat::Rgba
@ -261,6 +291,7 @@ fn audio_pipeline(source: gst::Element, feed: &CaptureFeed) -> Result<gst::Pipel
.build();
let level = feed.audio.clone();
let received = feed.received_audio.clone();
let tx = feed.audio_tx.clone();
sink.set_callbacks(
gstreamer_app::AppSinkCallbacks::builder()
.new_sample(move |sink| {
@ -273,15 +304,22 @@ fn audio_pipeline(source: gst::Element, feed: &CaptureFeed) -> Result<gst::Pipel
if map.len() > 48_000 * 8 {
return Err(gst::FlowError::Error);
}
let mut samples = Vec::with_capacity(map.len() / 4);
let mut sum = 0.0f64;
let mut count = 0u32;
for bytes in map.as_slice().as_chunks::<4>().0 {
let value = f32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
if value.is_finite() {
samples.push(value);
sum += f64::from(value) * f64::from(value);
count += 1;
}
}
let _ = tx.send(Arc::new(super::AudioFrame {
samples,
rate: 48_000,
channels: 2,
}));
received.store(true, Ordering::Relaxed);
*level.lock().map_err(|_| gst::FlowError::Error)? = if count == 0 {
0.0
@ -324,12 +362,12 @@ mod tests {
#[test]
fn silent_unavailable_source_times_out_and_drop_clears_preview() {
let feed = CaptureFeed::default();
*feed.video.lock().expect("frame lock") = Some(VideoFrame {
*feed.video.lock().expect("frame lock") = Some(Arc::new(VideoFrame {
width: 1,
height: 1,
rgba: vec![1, 2, 3, 255],
timestamp_ns: None,
});
}));
*feed.audio.lock().expect("level lock") = 0.5;
let session = CaptureSession {
pipelines: Vec::new(),

245
src/media/gateway_tests.rs Normal file
View file

@ -0,0 +1,245 @@
//! Synthetic full native-client → authenticated gateway → peer media fixture.
use super::{
AudioFrame, CaptureFeed, VideoFrame,
broadcast::BroadcastManager,
peer::{AudioTrack, Tracks},
};
use crate::{
identity,
net::{self, NetCommand, NetEvent, NetHandle},
};
use std::{
path::PathBuf,
process::{Child, Command, Stdio},
time::{Duration, Instant},
};
struct Gateway {
child: Child,
directory: PathBuf,
}
impl Drop for Gateway {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
let _ = std::fs::remove_dir_all(&self.directory);
}
}
async fn wait(handle: &NetHandle, test: impl Fn(&NetEvent) -> bool) -> NetEvent {
tokio::time::timeout(Duration::from_secs(8), async {
loop {
if let Some(event) = handle.try_recv() {
if let NetEvent::Disconnected { reason, .. } = &event {
panic!("fixture disconnected: {reason}");
}
if let NetEvent::Error { message, .. } = &event {
panic!("gateway rejected fixture: {message}");
}
if test(&event) {
return event;
}
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("authenticated gateway fixture timeout")
}
async fn connect(handle: &NetHandle, address: &str) -> String {
handle
.send(NetCommand::Connect {
address: address.into(),
auto_reconnect: false,
})
.await;
let NetEvent::Connected { session_id, .. } =
wait(handle, |event| matches!(event, NetEvent::Connected { .. })).await
else {
panic!("connected event")
};
handle
.send(NetCommand::JoinChannel {
channel_id: "voice".into(),
})
.await;
wait(
handle,
|event| matches!(event, NetEvent::ChannelState { kind, .. } if kind=="voice"),
)
.await;
// Browsing text must preserve the independently authorized voice membership.
handle
.send(NetCommand::JoinChannel {
channel_id: "general".into(),
})
.await;
tokio::time::sleep(Duration::from_millis(20)).await;
session_id
}
async fn pump(handle: &NetHandle, manager: &mut BroadcastManager) {
while let Some(event) = handle.try_recv() {
match event {
NetEvent::MediaEvent(event) => manager.on_event(event).expect("authorized media event"),
NetEvent::Error { message, .. } => panic!("gateway rejected media fixture: {message}"),
NetEvent::Disconnected { reason, .. } => {
panic!("gateway fixture disconnected: {reason}")
}
_ => {}
}
}
manager.tick();
if let Some(error) = manager.take_error() {
panic!("media fixture failed: {error}");
}
while let Some(control) = manager.take_control() {
handle.send(NetCommand::MediaControl(control)).await;
}
}
#[tokio::test]
#[ignore = "requires freshly built current VNOX_GATEWAY_BIN; uses synthetic media only"]
async fn gateway_authorizes_real_two_peer_video_and_application_audio() {
let binary = std::env::var_os("VNOX_GATEWAY_BIN").expect("set VNOX_GATEWAY_BIN");
let directory =
std::env::temp_dir().join(format!("vnox-media-gateway-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&directory).expect("fixture dir");
let port = std::net::TcpListener::bind("127.0.0.1:0")
.expect("port")
.local_addr()
.expect("address")
.port();
let voice_port = std::net::UdpSocket::bind("127.0.0.1:0")
.expect("UDP port")
.local_addr()
.expect("UDP address")
.port();
let config = format!(
r#"
[node]
name="media-fixture"
address="127.0.0.1"
[gateway]
bind="127.0.0.1:{port}"
admin_bind="127.0.0.1:0"
message_rate_per_sec=0
[voice]
bind="127.0.0.1:{voice_port}"
[storage]
data_dir="{}"
backend="sqlite"
sqlite_path="{}/fixture.db"
[server]
mode="private"
"#,
directory.display(),
directory.display()
);
let config_path = directory.join("config.toml");
std::fs::write(&config_path, config).expect("config");
let child = Command::new(binary)
.arg("--config")
.arg(config_path)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("start actual gateway");
let _gateway = Gateway { child, directory };
tokio::time::sleep(Duration::from_millis(700)).await;
let publisher_net = net::spawn(identity::generate("Synthetic publisher"));
let viewer_net = net::spawn(identity::generate("Synthetic viewer"));
let address = format!("127.0.0.1:{port}");
let publisher_session = connect(&publisher_net, &address).await;
let viewer_session = connect(&viewer_net, &address).await;
let mut publisher = BroadcastManager::default();
publisher.set_scope(&publisher_session, Some("voice"));
let mut viewer = BroadcastManager::default();
viewer.set_scope(&viewer_session, Some("voice"));
let feed = CaptureFeed::default();
let share = publisher
.publish(
&feed,
Tracks {
application_audio: true,
microphone: false,
},
)
.expect("explicit publish");
let start = Instant::now();
let mut samples = 0usize;
let mut requested = false;
let mut received_pcm = Vec::new();
let mut received_video = false;
while start.elapsed() < Duration::from_secs(15) {
feed.fixture_video(VideoFrame {
width: 64,
height: 48,
rgba: [200, 50, 70, 255].repeat(64 * 48),
timestamp_ns: None,
});
let pcm = (samples..samples + 960)
.flat_map(|n| {
let sample = (n as f32 * 440.0 * std::f32::consts::TAU / 48_000.0).sin() * 0.3;
[sample, sample]
})
.collect();
samples += 960;
feed.fixture_audio(AudioFrame {
samples: pcm,
rate: 48_000,
channels: 2,
});
pump(&publisher_net, &mut publisher).await;
pump(&viewer_net, &mut viewer).await;
if !requested && viewer.shares().iter().any(|item| item.id == share) {
viewer.watch(&share).expect("explicit watch");
requested = true;
}
if let Some(remote) = viewer.remote_feed(&share) {
if let Some(frame) = remote.take_video() {
assert_eq!((frame.width, frame.height), (64, 48));
received_video = true;
}
while let Some(pcm) = remote.take_audio(AudioTrack::Application) {
received_pcm.extend(pcm.samples.into_iter().step_by(2));
}
assert_eq!(
remote
.microphone_samples
.load(std::sync::atomic::Ordering::Relaxed),
0,
"unrequested microphone not delivered"
);
}
if received_video && received_pcm.len() > 24_000 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert!(
publisher.publishing() && publisher.viewer_count() == 1,
"gateway authorized connected viewer"
);
assert!(
received_video && received_pcm.len() > 24_000,
"actual decoded video and Opus reached consenting viewer"
);
let frequency = received_pcm
.windows(2)
.filter(|p| p[0] < 0.0 && p[1] >= 0.0)
.count() as f32
* 48_000.0
/ received_pcm.len() as f32;
assert!(
(frequency - 440.0).abs() < 60.0,
"actual selected fixture audio: {frequency} Hz"
);
publisher.stop_publishing();
pump(&publisher_net, &mut publisher).await;
let stop = Instant::now();
while viewer.is_watching(&share) && stop.elapsed() < Duration::from_secs(3) {
pump(&viewer_net, &mut viewer).await;
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert!(
!viewer.is_watching(&share),
"authenticated STOP revoked receiver transport"
);
}

View file

@ -5,7 +5,7 @@
mod policy;
mod sources;
pub use policy::{AudioChoice, AudioKind, AudioSource, CaptureOptions, VideoFrame};
pub use policy::{AudioChoice, AudioFrame, AudioKind, AudioSource, CaptureOptions, VideoFrame};
pub use sources::{audio_sources, parse_audio_sources, validate_audio_choice};
#[cfg(all(target_os = "linux", feature = "screen-capture"))]
@ -29,3 +29,11 @@ pub fn unavailable_reason() -> Option<&'static str> {
)
}
}
#[cfg(all(target_os = "linux", feature = "screen-capture"))]
pub mod broadcast;
#[cfg(all(target_os = "linux", feature = "screen-capture"))]
pub mod peer;
#[cfg(all(test, target_os = "linux", feature = "screen-capture"))]
mod gateway_tests;

646
src/media/peer.rs Normal file
View file

@ -0,0 +1,646 @@
//! Established ICE/DTLS-SRTP transport via GStreamer webrtcbin, with independent tracks.
use super::{AudioFrame, VideoFrame};
use gst::prelude::*;
use gstreamer as gst;
use gstreamer_webrtc as rtc;
use std::sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
mpsc,
};
#[derive(Clone, Copy, Debug, Default)]
pub struct Tracks {
pub application_audio: bool,
pub microphone: bool,
}
#[derive(Debug)]
pub enum Signal {
Offer(String),
Answer(String),
Ice { mline: u32, candidate: String },
Error(String),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AudioTrack {
Application,
Microphone,
}
#[derive(Clone, Default)]
pub struct RemoteFeed {
video: Arc<Mutex<Option<Arc<VideoFrame>>>>,
app: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
mic: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
pub video_frames: Arc<AtomicU64>,
pub application_samples: Arc<AtomicU64>,
pub microphone_samples: Arc<AtomicU64>,
}
impl RemoteFeed {
pub fn take_video(&self) -> Option<Arc<VideoFrame>> {
self.video.lock().ok()?.take()
}
pub fn take_audio(&self, track: AudioTrack) -> Option<AudioFrame> {
match track {
AudioTrack::Application => self.app.lock().ok()?.pop_front(),
AudioTrack::Microphone => self.mic.lock().ok()?.pop_front(),
}
}
}
/// Explicit administrator-configured ICE services. Empty means host candidates only.
#[derive(Clone, Debug, Default)]
pub struct IceServers {
pub stun: Option<String>,
pub turn: Option<String>,
}
impl IceServers {
pub fn validate(&self) -> Result<(), String> {
if self.stun.as_ref().is_some_and(|uri| {
!uri.starts_with("stun://")
|| uri.len() > 2048
|| uri.len() <= 7
|| uri.chars().any(char::is_whitespace)
}) || self.turn.as_ref().is_some_and(|uri| {
!(uri.starts_with("turn://") || uri.starts_with("turns://"))
|| uri.len() > 2048
|| uri.len() <= 7
|| uri.chars().any(char::is_whitespace)
}) {
return Err("Invalid explicitly configured ICE server URI".into());
}
Ok(())
}
}
/// One directed publisher/viewer connection. The authenticated gateway routes its signals.
pub struct Peer {
pipeline: gst::Pipeline,
rtc: gst::Element,
video: Option<gstreamer_app::AppSrc>,
app: Option<gstreamer_app::AppSrc>,
mic: Option<gstreamer_app::AppSrc>,
events: mpsc::Receiver<Signal>,
tx: mpsc::SyncSender<Signal>,
pub remote: RemoteFeed,
remote_set: bool,
started: std::time::Instant,
pending_ice: Vec<(u32, String)>,
app_time: Mutex<Option<u64>>,
mic_time: Mutex<Option<u64>>,
}
pub fn check_backend() -> Result<(), String> {
gst::init().map_err(error)?;
for plugin in [
"webrtcbin",
"nicesrc",
"nicesink",
"dtlssrtpenc",
"dtlssrtpdec",
"vp8enc",
"vp8dec",
"rtpvp8pay",
"rtpvp8depay",
"opusenc",
"opusdec",
"rtpopuspay",
"rtpopusdepay",
] {
if gst::ElementFactory::find(plugin).is_none() {
return Err(format!("Missing WebRTC plugin: {plugin}"));
}
}
Ok(())
}
impl Peer {
pub fn publisher(tracks: Tracks) -> Result<Self, String> {
Self::new(true, tracks, &IceServers::default())
}
pub fn viewer(tracks: Tracks) -> Result<Self, String> {
Self::new(false, tracks, &IceServers::default())
}
pub fn publisher_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
Self::new(true, tracks, ice)
}
pub fn viewer_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
Self::new(false, tracks, ice)
}
fn new(publisher: bool, tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
ice.validate()?;
check_backend()?;
let pipeline = gst::Pipeline::new();
let rtc = gst::ElementFactory::make("webrtcbin")
.name("transport")
.build()
.map_err(error)?;
rtc.set_property_from_str("bundle-policy", "max-bundle");
rtc.set_property("latency", 100u32);
if let Some(uri) = &ice.stun {
rtc.set_property("stun-server", uri);
}
if let Some(uri) = &ice.turn {
rtc.set_property("turn-server", uri);
}
pipeline.add(&rtc).map_err(error)?;
let (tx, events) = mpsc::sync_channel(64);
let ice_tx = tx.clone();
rtc.connect("on-ice-candidate", false, move |values| {
if let (Ok(mline), Ok(candidate)) = (values[1].get::<u32>(), values[2].get::<String>())
{
let _ = ice_tx.try_send(Signal::Ice { mline, candidate });
}
None
});
let remote = RemoteFeed::default();
if !publisher {
let weak_pipeline = pipeline.downgrade();
let feed = remote.clone();
let receive_tx = tx.clone();
rtc.connect_pad_added(move |_, pad| {
if pad.direction() != gst::PadDirection::Src {
return;
}
if let Some(pipeline) = weak_pipeline.upgrade()
&& let Err(error) = receive(&pipeline, pad, &feed, tracks)
{
let _ = receive_tx.try_send(Signal::Error(error));
}
});
}
let video = if publisher {
Some(add_sender(&pipeline, &rtc, 96, "screen-video", true)?)
} else {
None
};
let app = if publisher && tracks.application_audio {
Some(add_sender(&pipeline, &rtc, 97, "application-audio", false)?)
} else {
None
};
let mic = if publisher && tracks.microphone {
Some(add_sender(&pipeline, &rtc, 98, "microphone", false)?)
} else {
None
};
pipeline.set_state(gst::State::Playing).map_err(error)?;
Ok(Self {
pipeline,
rtc,
video,
app,
mic,
events,
tx,
remote,
remote_set: false,
started: std::time::Instant::now(),
pending_ice: Vec::new(),
app_time: Mutex::new(None),
mic_time: Mutex::new(None),
})
}
/// Create an SDP offer only after the gateway authorizes a viewer subscription.
pub fn offer(&self) {
create_description(&self.rtc, true, self.signal_sender());
}
fn signal_sender(&self) -> mpsc::SyncSender<Signal> {
self.tx.clone()
}
pub fn apply_offer(&mut self, sdp: &str) -> Result<(), String> {
self.set_description(sdp, true)?;
create_description(&self.rtc, false, self.signal_sender());
Ok(())
}
pub fn apply_answer(&mut self, sdp: &str) -> Result<(), String> {
self.set_description(sdp, false)
}
fn set_description(&mut self, text: &str, offer: bool) -> Result<(), String> {
if text.len() > 65_536 {
return Err("SDP exceeds limit".into());
}
let sdp = gstreamer_sdp::SDPMessage::parse_buffer(text.as_bytes()).map_err(error)?;
let description = rtc::WebRTCSessionDescription::new(
if offer {
rtc::WebRTCSDPType::Offer
} else {
rtc::WebRTCSDPType::Answer
},
sdp,
);
let promise = gst::Promise::new();
self.rtc
.emit_by_name::<()>("set-remote-description", &[&description, &promise]);
if promise.wait() != gst::PromiseResult::Replied {
return Err("Remote SDP was rejected".into());
}
if let Some(reply) = promise.get_reply()
&& reply.has_field("error")
{
return Err("Remote SDP was rejected".into());
}
self.remote_set = true;
for (mline, candidate) in self.pending_ice.drain(..) {
self.rtc
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
}
Ok(())
}
pub fn add_ice(&mut self, mline: u32, candidate: &str) -> Result<(), String> {
if mline > 2 || candidate.len() > 2048 || self.pending_ice.len() >= 64 {
return Err("ICE candidate exceeds limit".into());
}
if self.remote_set {
self.rtc
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
} else {
self.pending_ice.push((mline, candidate.into()));
}
Ok(())
}
pub fn poll(&self) -> Result<Option<Signal>, String> {
if let Some(bus) = self.pipeline.bus() {
while let Some(message) = bus.pop() {
match message.view() {
gst::MessageView::Error(e) => return Err(format!("WebRTC: {}", e.error())),
gst::MessageView::Eos(..) => return Err("WebRTC stream closed".into()),
_ => {}
}
}
}
let state = self
.rtc
.property::<rtc::WebRTCPeerConnectionState>("connection-state");
if self.started.elapsed() > std::time::Duration::from_secs(20)
&& state != rtc::WebRTCPeerConnectionState::Connected
{
return Err("WebRTC connection timed out".into());
}
if matches!(
state,
rtc::WebRTCPeerConnectionState::Failed | rtc::WebRTCPeerConnectionState::Closed
) {
return Err("WebRTC peer disconnected".into());
}
Ok(self.events.try_recv().ok())
}
pub fn connected(&self) -> bool {
self.rtc
.property::<rtc::WebRTCPeerConnectionState>("connection-state")
== rtc::WebRTCPeerConnectionState::Connected
}
pub fn push_video(&self, frame: &VideoFrame) -> Result<(), String> {
frame.validate()?;
let source = self.video.as_ref().ok_or("Peer does not publish video")?;
source.set_caps(Some(
&gst::Caps::builder("video/x-raw")
.field("format", "RGBA")
.field("width", frame.width as i32)
.field("height", frame.height as i32)
.field("framerate", gst::Fraction::new(30, 1))
.build(),
));
source
.push_buffer(gst::Buffer::from_mut_slice(frame.rgba.clone()))
.map_err(error)?;
Ok(())
}
pub fn push_audio(&self, track: AudioTrack, frame: &AudioFrame) -> Result<(), String> {
frame.validate()?;
let source = match track {
AudioTrack::Application => &self.app,
AudioTrack::Microphone => &self.mic,
}
.as_ref()
.ok_or("Audio track was not explicitly enabled")?;
source.set_caps(Some(
&gst::Caps::builder("audio/x-raw")
.field("format", "F32LE")
.field("layout", "interleaved")
.field("rate", frame.rate as i32)
.field("channels", i32::from(frame.channels))
.build(),
));
let bytes: Vec<u8> = frame
.samples
.iter()
.flat_map(|sample| sample.to_le_bytes())
.collect();
let clock = match track {
AudioTrack::Application => &self.app_time,
AudioTrack::Microphone => &self.mic_time,
};
let mut next = clock
.lock()
.map_err(|_| "Audio timestamp clock unavailable")?;
let pts = next.unwrap_or_else(|| {
self.pipeline
.current_running_time()
.map(|time| time.nseconds())
.unwrap_or(0)
});
let duration = (frame.samples.len() as u64 / u64::from(frame.channels)) * 1_000_000_000
/ u64::from(frame.rate);
let mut buffer = gst::Buffer::from_mut_slice(bytes);
if let Some(buffer) = buffer.get_mut() {
buffer.set_pts(gst::ClockTime::from_nseconds(pts));
buffer.set_duration(gst::ClockTime::from_nseconds(duration));
}
*next = Some(pts.saturating_add(duration));
source.push_buffer(buffer).map_err(error)?;
Ok(())
}
}
impl Drop for Peer {
fn drop(&mut self) {
let _ = self.pipeline.set_state(gst::State::Null);
}
}
fn add_sender(
pipeline: &gst::Pipeline,
rtc: &gst::Element,
payload: i32,
msid: &str,
video: bool,
) -> Result<gstreamer_app::AppSrc, String> {
let description = if video {
format!(
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=8388608 leaky-type=downstream ! queue max-size-buffers=2 leaky=downstream ! videoconvert ! vp8enc deadline=1 cpu-used=8 target-bitrate=1500000 keyframe-max-dist=60 ! rtpvp8pay pt={payload} ! capsfilter caps=\"application/x-rtp,media=video,encoding-name=VP8,payload={payload},clock-rate=90000\""
)
} else {
format!(
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=384000 leaky-type=downstream ! queue max-size-buffers=8 leaky=downstream ! audioconvert ! audioresample ! opusenc bitrate=64000 ! rtpopuspay pt={payload} ! capsfilter caps=\"application/x-rtp,media=audio,encoding-name=OPUS,payload={payload},clock-rate=48000\""
)
};
let bin = gst::parse::bin_from_description(&description, true).map_err(error)?;
let source = bin
.by_name("source")
.ok_or("Missing media appsrc")?
.downcast::<gstreamer_app::AppSrc>()
.map_err(|_| "Wrong media source type")?;
pipeline.add(&bin).map_err(error)?;
let src = bin.static_pad("src").ok_or("Missing RTP source pad")?;
let sink = rtc
.request_pad_simple("sink_%u")
.ok_or("WebRTC refused a media track")?;
sink.set_property("msid", msid);
let transceiver = sink.property::<rtc::WebRTCRTPTransceiver>("transceiver");
transceiver.set_property("direction", rtc::WebRTCRTPTransceiverDirection::Sendonly);
src.link(&sink).map_err(error)?;
Ok(source)
}
fn create_description(webrtc: &gst::Element, offer: bool, tx: mpsc::SyncSender<Signal>) {
let weak = webrtc.downgrade();
let promise = gst::Promise::with_change_func(move |reply| {
let result = (|| {
let reply = reply
.map_err(|e| format!("SDP promise failed: {e:?}"))?
.ok_or("No SDP reply")?;
let description = reply
.get::<rtc::WebRTCSessionDescription>(if offer { "offer" } else { "answer" })
.map_err(error)?;
let webrtc = weak.upgrade().ok_or("WebRTC peer closed")?;
webrtc.emit_by_name::<()>(
"set-local-description",
&[&description, &None::<gst::Promise>],
);
let text = description.sdp().as_text().map_err(error)?;
Ok::<_, String>(if offer {
Signal::Offer(text)
} else {
Signal::Answer(text)
})
})();
let _ = tx.try_send(result.unwrap_or_else(Signal::Error));
});
webrtc.emit_by_name::<()>(
if offer {
"create-offer"
} else {
"create-answer"
},
&[&None::<gst::Structure>, &promise],
);
}
fn receive(
pipeline: &gst::Pipeline,
pad: &gst::Pad,
feed: &RemoteFeed,
tracks: Tracks,
) -> Result<(), String> {
let caps = pad.current_caps().ok_or("Incoming track has no caps")?;
let payload = caps
.structure(0)
.ok_or("Incoming track has no format")?
.get::<i32>("payload")
.map_err(error)?;
let video = payload == 96;
if !(video || payload == 97 && tracks.application_audio || payload == 98 && tracks.microphone) {
return Err("Unannounced remote track rejected".into());
}
let description = if video {
"queue max-size-buffers=2 leaky=downstream ! rtpvp8depay ! vp8dec ! videoconvert ! video/x-raw,format=RGBA,width=[1,1920],height=[1,1080] ! appsink name=sink sync=false max-buffers=1 drop=true"
} else {
"queue max-size-buffers=8 leaky=downstream ! rtpopusdepay ! opusdec ! audioconvert ! audioresample ! audio/x-raw,format=F32LE,layout=interleaved,rate=48000,channels=2 ! appsink name=sink sync=false max-buffers=4 drop=true"
};
let bin = gst::parse::bin_from_description(description, true).map_err(error)?;
let sink = bin
.by_name("sink")
.ok_or("Remote sink missing")?
.downcast::<gstreamer_app::AppSink>()
.map_err(|_| "Wrong remote sink")?;
let feed = feed.clone();
sink.set_callbacks(
gstreamer_app::AppSinkCallbacks::builder()
.new_sample(move |sink| {
let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
if video {
let frame =
super::capture::copy_video(&sample).map_err(|_| gst::FlowError::Error)?;
*feed.video.lock().map_err(|_| gst::FlowError::Error)? = Some(Arc::new(frame));
feed.video_frames.fetch_add(1, Ordering::Relaxed);
} else {
let map = sample
.buffer()
.ok_or(gst::FlowError::Error)?
.map_readable()
.map_err(|_| gst::FlowError::Error)?;
if map.len() > 384000 {
return Err(gst::FlowError::Error);
}
let samples: Vec<f32> = map
.as_slice()
.as_chunks::<4>()
.0
.iter()
.map(|bytes| f32::from_le_bytes(*bytes))
.collect();
let count = samples.len() as u64;
let frame = AudioFrame {
samples,
rate: 48000,
channels: 2,
};
frame.validate().map_err(|_| gst::FlowError::Error)?;
let (queue, received) = if payload == 97 {
(&feed.app, &feed.application_samples)
} else {
(&feed.mic, &feed.microphone_samples)
};
let mut queue = queue.lock().map_err(|_| gst::FlowError::Error)?;
if queue.len() == 8 {
queue.pop_front();
}
queue.push_back(frame);
received.fetch_add(count, Ordering::Relaxed);
}
Ok(gst::FlowSuccess::Ok)
})
.build(),
);
pipeline.add(&bin).map_err(error)?;
bin.sync_state_with_parent().map_err(error)?;
pad.link(&bin.static_pad("sink").ok_or("Missing remote RTP sink")?)
.map_err(error)?;
Ok(())
}
fn error(error: impl std::fmt::Display) -> String {
error.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{Duration, Instant};
fn forward(from: &Peer, to: &mut Peer) {
while let Some(signal) = from.poll().expect("healthy peer") {
match signal {
Signal::Offer(sdp) => to.apply_offer(&sdp).expect("offer accepted"),
Signal::Answer(sdp) => to.apply_answer(&sdp).expect("answer accepted"),
Signal::Ice { mline, candidate } => {
to.add_ice(mline, &candidate).expect("candidate accepted")
}
Signal::Error(error) => panic!("signalling failed: {error}"),
}
}
}
#[test]
fn two_peers_deliver_video_and_separate_application_and_microphone_tracks() {
let tracks = Tracks {
application_audio: true,
microphone: true,
};
let mut publisher = Peer::publisher(tracks).expect("publisher");
let mut viewer = Peer::viewer(tracks).expect("viewer");
let frame = VideoFrame {
width: 64,
height: 48,
rgba: [200, 50, 70, 255].repeat(64 * 48),
timestamp_ns: None,
};
publisher.push_video(&frame).expect("fixture video");
publisher.offer();
let start = Instant::now();
let mut index = 0usize;
let mut app_pcm = Vec::new();
let mut mic_pcm = Vec::new();
while start.elapsed() < Duration::from_secs(12) {
forward(&publisher, &mut viewer);
forward(&viewer, &mut publisher);
publisher.push_video(&frame).expect("video push");
for (track, frequency) in [
(AudioTrack::Application, 440.0f32),
(AudioTrack::Microphone, 880.0f32),
] {
let samples = (index..index + 960)
.flat_map(|sample| {
let value = (sample as f32 * frequency * std::f32::consts::TAU / 48_000.0)
.sin()
* 0.3;
[value, value]
})
.collect();
publisher
.push_audio(
track,
&AudioFrame {
samples,
rate: 48_000,
channels: 2,
},
)
.expect("independent audio push");
}
index += 960;
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Application) {
app_pcm.extend(frame.samples.into_iter().step_by(2));
}
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Microphone) {
mic_pcm.extend(frame.samples.into_iter().step_by(2));
}
if viewer.remote.video_frames.load(Ordering::Relaxed) >= 3
&& app_pcm.len() > 24_000
&& mic_pcm.len() > 24_000
{
break;
}
std::thread::sleep(Duration::from_millis(20));
}
assert!(
publisher.connected() && viewer.connected(),
"DTLS/ICE connection established"
);
let received = viewer
.remote
.take_video()
.expect("actual decoded remote video");
assert_eq!((received.width, received.height), (64, 48));
assert!((i32::from(received.rgba[0]) - 200).abs() < 20);
assert!(
app_pcm.len() > 24_000 && mic_pcm.len() > 24_000,
"both distinct Opus tracks delivered"
);
let frequency = |pcm: &[f32]| {
pcm.windows(2)
.filter(|pair| pair[0] < 0.0 && pair[1] >= 0.0)
.count() as f32
* 48_000.0
/ pcm.len() as f32
};
let app_frequency = frequency(&app_pcm);
let mic_frequency = frequency(&mic_pcm);
assert!(
(app_frequency - 440.0).abs() < 60.0,
"app frequency {app_frequency}"
);
assert!(
(mic_frequency - 880.0).abs() < 100.0,
"microphone frequency {mic_frequency}"
);
assert!(
publisher
.push_audio(
AudioTrack::Application,
&AudioFrame {
samples: vec![f32::NAN],
rate: 48_000,
channels: 1
}
)
.is_err()
);
}
#[test]
fn unrequested_audio_track_cannot_be_enabled_by_push() {
let publisher = Peer::publisher(Tracks::default()).expect("publisher");
let pcm = AudioFrame {
samples: vec![0.0; 960],
rate: 48_000,
channels: 1,
};
assert!(publisher.push_audio(AudioTrack::Microphone, &pcm).is_err());
assert!(publisher.push_audio(AudioTrack::Application, &pcm).is_err());
}
}

View file

@ -107,3 +107,28 @@ mod tests {
);
}
}
/// One bounded independent PCM source, ready for an Opus track.
#[derive(Clone, Debug)]
pub struct AudioFrame {
pub samples: Vec<f32>,
pub rate: u32,
pub channels: u16,
}
impl AudioFrame {
pub fn validate(&self) -> Result<(), String> {
if self.rate != 48_000
|| !(1..=2).contains(&self.channels)
|| self.samples.len() > 48_000 * 2
|| self.samples.is_empty()
|| !self
.samples
.len()
.is_multiple_of(usize::from(self.channels))
|| self.samples.iter().any(|sample| !sample.is_finite())
{
return Err("Invalid or unbounded PCM frame".into());
}
Ok(())
}
}