feat(client): protocol v2 wire layer, screen sharing, channel webhooks and secure voice
Verified against the real gateway via tests/gateway_interop.rs.
This commit is contained in:
parent
8460fb28fd
commit
3d7dd5413b
65 changed files with 6048 additions and 1156 deletions
575
src/media/broadcast.rs
Normal file
575
src/media/broadcast.rs
Normal file
|
|
@ -0,0 +1,575 @@
|
|||
//! UI-independent broadcast controller. User actions enqueue authenticated LNEx controls.
|
||||
use super::{
|
||||
AudioFrame, CaptureFeed, VideoFrame,
|
||||
peer::{AudioTrack, IceServers, Peer, RemoteFeed, Signal, Tracks},
|
||||
};
|
||||
use crate::net::wire::{MediaControl, MediaEvent};
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
const START: u32 = 1;
|
||||
const STOP: u32 = 2;
|
||||
const WATCH: u32 = 3;
|
||||
const UNWATCH: u32 = 4;
|
||||
const OFFER: u32 = 5;
|
||||
const ANSWER: u32 = 6;
|
||||
const ICE: u32 = 7;
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AvailableShare {
|
||||
pub id: String,
|
||||
pub publisher_session: String,
|
||||
pub publisher_user: String,
|
||||
pub application_audio: bool,
|
||||
pub microphone: bool,
|
||||
}
|
||||
struct Publisher {
|
||||
id: String,
|
||||
tracks: Tracks,
|
||||
acknowledged: bool,
|
||||
started: std::time::Instant,
|
||||
video: broadcast::Receiver<Arc<VideoFrame>>,
|
||||
audio: broadcast::Receiver<Arc<AudioFrame>>,
|
||||
latest: Option<Arc<VideoFrame>>,
|
||||
peers: HashMap<String, Peer>,
|
||||
}
|
||||
struct Viewer {
|
||||
share: AvailableShare,
|
||||
peer: Option<Peer>,
|
||||
pending_ice: Vec<(u32, String)>,
|
||||
started: std::time::Instant,
|
||||
}
|
||||
#[derive(Default)]
|
||||
pub struct BroadcastManager {
|
||||
session: String,
|
||||
channel: Option<String>,
|
||||
available: HashMap<String, AvailableShare>,
|
||||
publisher: Option<Publisher>,
|
||||
viewers: HashMap<String, Viewer>,
|
||||
outgoing: VecDeque<MediaControl>,
|
||||
errors: VecDeque<String>,
|
||||
ice: IceServers,
|
||||
}
|
||||
impl BroadcastManager {
|
||||
pub fn configure_ice(&mut self, ice: IceServers) -> Result<(), String> {
|
||||
ice.validate()?;
|
||||
if self.publisher.is_some() || !self.viewers.is_empty() {
|
||||
return Err("Stop broadcasts before changing ICE services".into());
|
||||
}
|
||||
self.ice = ice;
|
||||
Ok(())
|
||||
}
|
||||
/// Called after authenticated session/voice membership changes; revokes local peers first.
|
||||
pub fn set_scope(&mut self, session: &str, channel: Option<&str>) {
|
||||
if self.session == session && self.channel.as_deref() == channel {
|
||||
return;
|
||||
}
|
||||
self.stop();
|
||||
self.outgoing.clear();
|
||||
self.available.clear();
|
||||
self.session = session.into();
|
||||
self.channel = channel.map(str::to_owned);
|
||||
if self.channel.is_some() {
|
||||
self.enqueue(WATCH, "", "", "", 0, Tracks::default());
|
||||
}
|
||||
}
|
||||
/// Explicit user publication only. A portal preview alone never calls this method.
|
||||
pub fn publish(&mut self, feed: &CaptureFeed, tracks: Tracks) -> Result<String, String> {
|
||||
if self.channel.is_none() || self.session.is_empty() {
|
||||
return Err("Join a voice channel before broadcasting".into());
|
||||
}
|
||||
if self.publisher.is_some() {
|
||||
return Err("A broadcast is already active".into());
|
||||
}
|
||||
if self.outgoing.len() >= 120 {
|
||||
return Err(
|
||||
"Media signalling congested; wait before starting another broadcast".into(),
|
||||
);
|
||||
}
|
||||
super::peer::check_backend()?;
|
||||
let id = uuid::Uuid::new_v4().to_string();
|
||||
self.publisher = Some(Publisher {
|
||||
id: id.clone(),
|
||||
tracks,
|
||||
acknowledged: false,
|
||||
started: std::time::Instant::now(),
|
||||
video: feed.subscribe_video(),
|
||||
audio: feed.subscribe_audio(),
|
||||
latest: None,
|
||||
peers: HashMap::new(),
|
||||
});
|
||||
self.enqueue(START, &id, "", "", 0, tracks);
|
||||
Ok(id)
|
||||
}
|
||||
pub fn stop_publishing(&mut self) {
|
||||
if let Some(publisher) = self.publisher.take() {
|
||||
self.enqueue(STOP, &publisher.id, "", "", 0, publisher.tracks);
|
||||
}
|
||||
}
|
||||
/// Explicit receiver consent; no automatic incoming peer or speaker playback.
|
||||
pub fn watch(&mut self, id: &str) -> Result<(), String> {
|
||||
if self.viewers.contains_key(id) {
|
||||
return Ok(());
|
||||
}
|
||||
if self.outgoing.len() >= 120 {
|
||||
return Err(
|
||||
"Media signalling congested; wait before starting another broadcast".into(),
|
||||
);
|
||||
}
|
||||
if self.viewers.len() >= 4 {
|
||||
return Err("At most four broadcasts may be viewed at once".into());
|
||||
}
|
||||
let share = self
|
||||
.available
|
||||
.get(id)
|
||||
.ok_or("Broadcast is no longer available")?
|
||||
.clone();
|
||||
if share.publisher_session == self.session {
|
||||
return Err("Cannot watch your own broadcast".into());
|
||||
}
|
||||
self.viewers.insert(
|
||||
id.into(),
|
||||
Viewer {
|
||||
share,
|
||||
peer: None,
|
||||
pending_ice: Vec::new(),
|
||||
started: std::time::Instant::now(),
|
||||
},
|
||||
);
|
||||
self.enqueue(WATCH, id, "", "", 0, Tracks::default());
|
||||
Ok(())
|
||||
}
|
||||
pub fn unwatch(&mut self, id: &str) {
|
||||
if self.viewers.remove(id).is_some() {
|
||||
self.enqueue(UNWATCH, id, "", "", 0, Tracks::default());
|
||||
}
|
||||
}
|
||||
pub fn stop(&mut self) {
|
||||
self.stop_publishing();
|
||||
let ids: Vec<_> = self.viewers.keys().cloned().collect();
|
||||
for id in ids {
|
||||
self.unwatch(&id);
|
||||
}
|
||||
}
|
||||
pub fn shares(&self) -> Vec<AvailableShare> {
|
||||
let mut shares: Vec<_> = self.available.values().cloned().collect();
|
||||
shares.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
shares
|
||||
}
|
||||
pub fn is_watching(&self, id: &str) -> bool {
|
||||
self.viewers.contains_key(id)
|
||||
}
|
||||
pub fn publication_pending(&self) -> bool {
|
||||
self.publisher.is_some()
|
||||
}
|
||||
pub fn publishing(&self) -> bool {
|
||||
self.publisher
|
||||
.as_ref()
|
||||
.is_some_and(|publisher| publisher.acknowledged)
|
||||
}
|
||||
pub fn viewer_count(&self) -> usize {
|
||||
self.publisher
|
||||
.as_ref()
|
||||
.map(|p| p.peers.values().filter(|peer| peer.connected()).count())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
pub fn remote_feed(&self, id: &str) -> Option<RemoteFeed> {
|
||||
Some(self.viewers.get(id)?.peer.as_ref()?.remote.clone())
|
||||
}
|
||||
pub fn take_control(&mut self) -> Option<MediaControl> {
|
||||
self.outgoing.pop_front()
|
||||
}
|
||||
pub fn take_error(&mut self) -> Option<String> {
|
||||
self.errors.pop_front()
|
||||
}
|
||||
/// Existing voice capture supplies this only when explicitly enabled and unmuted.
|
||||
pub fn push_microphone(&self, frame: &AudioFrame) -> Result<(), String> {
|
||||
if let Some(publisher) = &self.publisher {
|
||||
if !publisher.tracks.microphone {
|
||||
return Ok(());
|
||||
}
|
||||
for peer in publisher.peers.values() {
|
||||
peer.push_audio(AudioTrack::Microphone, frame)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn on_event(&mut self, event: MediaEvent) -> Result<(), String> {
|
||||
if Some(event.channel_id.as_str()) != self.channel.as_deref()
|
||||
|| event.share_id.len() > 64
|
||||
|| event.sender_session_id.len() > 64
|
||||
{
|
||||
return Err("Media event is outside the active voice scope".into());
|
||||
}
|
||||
match event.action {
|
||||
START => {
|
||||
if self.available.len() >= 128 && !self.available.contains_key(&event.share_id) {
|
||||
return Err("Too many broadcast announcements".into());
|
||||
}
|
||||
let share = AvailableShare {
|
||||
id: event.share_id.clone(),
|
||||
publisher_session: event.sender_session_id.clone(),
|
||||
publisher_user: event.publisher_user_id,
|
||||
application_audio: event.application_audio,
|
||||
microphone: event.microphone,
|
||||
};
|
||||
if let Some(publisher) = &mut self.publisher
|
||||
&& publisher.id == event.share_id
|
||||
&& event.sender_session_id == self.session
|
||||
{
|
||||
publisher.acknowledged = true;
|
||||
}
|
||||
self.available.insert(event.share_id, share);
|
||||
}
|
||||
STOP => {
|
||||
if let Some(share) = self.available.get(&event.share_id)
|
||||
&& share.publisher_session != event.sender_session_id
|
||||
{
|
||||
return Err("Stop sender is not the publisher".into());
|
||||
}
|
||||
self.available.remove(&event.share_id);
|
||||
self.viewers.remove(&event.share_id);
|
||||
if self
|
||||
.publisher
|
||||
.as_ref()
|
||||
.is_some_and(|p| p.id == event.share_id)
|
||||
{
|
||||
self.publisher.take();
|
||||
}
|
||||
}
|
||||
WATCH => {
|
||||
if let Some(publisher) = &mut self.publisher
|
||||
&& publisher.id == event.share_id
|
||||
&& publisher.acknowledged
|
||||
&& event.sender_session_id != self.session
|
||||
{
|
||||
if publisher.peers.len() >= 4 {
|
||||
return Err("Viewer limit exceeded".into());
|
||||
}
|
||||
if !publisher.peers.contains_key(&event.sender_session_id) {
|
||||
let peer = Peer::publisher_with_ice(publisher.tracks, &self.ice)?;
|
||||
if let Some(frame) = &publisher.latest {
|
||||
peer.push_video(frame)?;
|
||||
}
|
||||
peer.offer();
|
||||
publisher.peers.insert(event.sender_session_id, peer);
|
||||
}
|
||||
}
|
||||
}
|
||||
UNWATCH => {
|
||||
if let Some(publisher) = &mut self.publisher
|
||||
&& publisher.id == event.share_id
|
||||
{
|
||||
publisher.peers.remove(&event.sender_session_id);
|
||||
}
|
||||
if self
|
||||
.viewers
|
||||
.get(&event.share_id)
|
||||
.is_some_and(|v| v.share.publisher_session == event.sender_session_id)
|
||||
{
|
||||
self.viewers.remove(&event.share_id);
|
||||
}
|
||||
}
|
||||
OFFER => {
|
||||
let viewer = self
|
||||
.viewers
|
||||
.get_mut(&event.share_id)
|
||||
.ok_or("Unrequested broadcast offer rejected")?;
|
||||
if viewer.share.publisher_session != event.sender_session_id
|
||||
|| viewer.peer.is_some()
|
||||
{
|
||||
return Err("Unexpected publisher or duplicate offer".into());
|
||||
}
|
||||
let mut peer = Peer::viewer_with_ice(
|
||||
Tracks {
|
||||
application_audio: viewer.share.application_audio,
|
||||
microphone: viewer.share.microphone,
|
||||
},
|
||||
&self.ice,
|
||||
)?;
|
||||
peer.apply_offer(&event.body)?;
|
||||
for (mline, candidate) in viewer.pending_ice.drain(..) {
|
||||
peer.add_ice(mline, &candidate)?;
|
||||
}
|
||||
viewer.peer = Some(peer);
|
||||
}
|
||||
ANSWER => {
|
||||
let publisher = self
|
||||
.publisher
|
||||
.as_mut()
|
||||
.filter(|p| p.id == event.share_id)
|
||||
.ok_or("No active publication for answer")?;
|
||||
publisher
|
||||
.peers
|
||||
.get_mut(&event.sender_session_id)
|
||||
.ok_or("Unsubscribed answer rejected")?
|
||||
.apply_answer(&event.body)?;
|
||||
}
|
||||
ICE => {
|
||||
if let Some(publisher) = self.publisher.as_mut().filter(|p| p.id == event.share_id)
|
||||
{
|
||||
publisher
|
||||
.peers
|
||||
.get_mut(&event.sender_session_id)
|
||||
.ok_or("Unsubscribed ICE rejected")?
|
||||
.add_ice(event.mline_index, &event.body)?;
|
||||
} else {
|
||||
let viewer = self
|
||||
.viewers
|
||||
.get_mut(&event.share_id)
|
||||
.filter(|v| v.share.publisher_session == event.sender_session_id)
|
||||
.ok_or("Unrequested ICE rejected")?;
|
||||
if let Some(peer) = &mut viewer.peer {
|
||||
peer.add_ice(event.mline_index, &event.body)?;
|
||||
} else if viewer.pending_ice.len() < 64
|
||||
&& event.body.len() <= 2048
|
||||
&& event.mline_index <= 2
|
||||
{
|
||||
viewer.pending_ice.push((event.mline_index, event.body));
|
||||
} else {
|
||||
return Err("Too many early ICE candidates".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => return Err("Unknown media event".into()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Call frequently on the owning UI thread; samples remain bounded and nonblocking.
|
||||
pub fn tick(&mut self) {
|
||||
if self.publisher.as_ref().is_some_and(|p| {
|
||||
!p.acknowledged && p.started.elapsed() > std::time::Duration::from_secs(10)
|
||||
}) {
|
||||
self.stop_publishing();
|
||||
self.errors
|
||||
.push_back("Broadcast authorization timed out".into());
|
||||
}
|
||||
let expired: Vec<_> = self
|
||||
.viewers
|
||||
.iter()
|
||||
.filter(|(_, v)| {
|
||||
v.peer.is_none() && v.started.elapsed() > std::time::Duration::from_secs(20)
|
||||
})
|
||||
.map(|(id, _)| id.clone())
|
||||
.collect();
|
||||
for id in expired {
|
||||
self.unwatch(&id);
|
||||
if self.errors.len() < 8 {
|
||||
self.errors
|
||||
.push_back("Broadcast connection timed out".into());
|
||||
}
|
||||
}
|
||||
|
||||
let mut controls = Vec::new();
|
||||
let mut failures = Vec::new();
|
||||
if let Some(publisher) = &mut self.publisher {
|
||||
let mut new_video = false;
|
||||
loop {
|
||||
match publisher.video.try_recv() {
|
||||
Ok(frame) => {
|
||||
publisher.latest = Some(frame);
|
||||
new_video = true;
|
||||
}
|
||||
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
let mut audio = Vec::new();
|
||||
loop {
|
||||
match publisher.audio.try_recv() {
|
||||
Ok(frame) => audio.push(frame),
|
||||
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
for (viewer, peer) in &publisher.peers {
|
||||
let result = (|| {
|
||||
if new_video && let Some(frame) = &publisher.latest {
|
||||
peer.push_video(frame)?;
|
||||
}
|
||||
for frame in &audio {
|
||||
if publisher.tracks.application_audio {
|
||||
peer.push_audio(AudioTrack::Application, frame)?;
|
||||
}
|
||||
}
|
||||
while let Some(signal) = peer.poll()? {
|
||||
controls.push((publisher.id.clone(), viewer.clone(), signal));
|
||||
}
|
||||
Ok::<_, String>(())
|
||||
})();
|
||||
if let Err(error) = result {
|
||||
failures.push(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (id, viewer) in &self.viewers {
|
||||
if let Some(peer) = &viewer.peer {
|
||||
loop {
|
||||
match peer.poll() {
|
||||
Ok(Some(signal)) => controls.push((
|
||||
id.clone(),
|
||||
viewer.share.publisher_session.clone(),
|
||||
signal,
|
||||
)),
|
||||
Ok(None) => break,
|
||||
Err(error) => {
|
||||
failures.push(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (id, target, signal) in controls {
|
||||
if !self
|
||||
.publisher
|
||||
.as_ref()
|
||||
.is_some_and(|publisher| publisher.id == id)
|
||||
&& !self.viewers.contains_key(&id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
match signal {
|
||||
Signal::Offer(body) => {
|
||||
self.enqueue(OFFER, &id, &target, &body, 0, Tracks::default())
|
||||
}
|
||||
Signal::Answer(body) => {
|
||||
self.enqueue(ANSWER, &id, &target, &body, 0, Tracks::default())
|
||||
}
|
||||
Signal::Ice { mline, candidate } => {
|
||||
self.enqueue(ICE, &id, &target, &candidate, mline, Tracks::default())
|
||||
}
|
||||
Signal::Error(error) => failures.push(error),
|
||||
}
|
||||
}
|
||||
if !failures.is_empty() {
|
||||
self.stop();
|
||||
for error in failures {
|
||||
if self.errors.len() < 8 {
|
||||
self.errors.push_back(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn enqueue(
|
||||
&mut self,
|
||||
action: u32,
|
||||
id: &str,
|
||||
target: &str,
|
||||
body: &str,
|
||||
mline_index: u32,
|
||||
tracks: Tracks,
|
||||
) {
|
||||
if let Some(channel) = &self.channel {
|
||||
if self.outgoing.len() >= 128 {
|
||||
if self.errors.len() < 8 {
|
||||
self.errors
|
||||
.push_back("Media signalling queue overflow; broadcast stopped".into());
|
||||
}
|
||||
let mut cleanup: Vec<_> = self
|
||||
.outgoing
|
||||
.iter()
|
||||
.filter(|control| matches!(control.action, STOP | UNWATCH))
|
||||
.map(|control| (control.action, control.share_id.clone()))
|
||||
.collect();
|
||||
if let Some(publisher) = self.publisher.take() {
|
||||
cleanup.push((STOP, publisher.id));
|
||||
}
|
||||
cleanup.extend(self.viewers.drain().map(|(id, _)| (UNWATCH, id)));
|
||||
if matches!(action, STOP | UNWATCH)
|
||||
&& !cleanup
|
||||
.iter()
|
||||
.any(|(kind, share)| *kind == action && share == id)
|
||||
{
|
||||
cleanup.push((action, id.into()));
|
||||
}
|
||||
cleanup.sort();
|
||||
cleanup.dedup();
|
||||
self.outgoing.clear();
|
||||
for (action, share_id) in cleanup {
|
||||
self.outgoing.push_back(MediaControl {
|
||||
action,
|
||||
channel_id: channel.clone(),
|
||||
share_id,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
self.outgoing.push_back(MediaControl {
|
||||
action,
|
||||
channel_id: channel.clone(),
|
||||
share_id: id.into(),
|
||||
target_session_id: target.into(),
|
||||
body: body.into(),
|
||||
mline_index,
|
||||
application_audio: tracks.application_audio,
|
||||
microphone: tracks.microphone,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn signalling_overflow_discards_stale_offers_and_retains_stop() {
|
||||
let mut manager = BroadcastManager::default();
|
||||
manager.set_scope("publisher", Some("voice"));
|
||||
let feed = CaptureFeed::default();
|
||||
let id = manager
|
||||
.publish(&feed, Tracks::default())
|
||||
.expect("explicit publish");
|
||||
manager.outgoing.clear(); // START already handed to authenticated networking.
|
||||
for _ in 0..129 {
|
||||
manager.enqueue(
|
||||
ICE,
|
||||
&id,
|
||||
"viewer",
|
||||
"candidate:fixture",
|
||||
0,
|
||||
Tracks::default(),
|
||||
);
|
||||
}
|
||||
assert!(!manager.publication_pending());
|
||||
let controls: Vec<_> = manager.outgoing.drain(..).collect();
|
||||
assert_eq!(controls.len(), 1);
|
||||
assert_eq!(controls[0].action, STOP);
|
||||
assert_eq!(controls[0].share_id, id);
|
||||
}
|
||||
#[test]
|
||||
fn overflow_preserves_queued_cleanup_for_previous_lifecycle() {
|
||||
let mut manager = BroadcastManager::default();
|
||||
manager.set_scope("publisher", Some("voice"));
|
||||
let feed = CaptureFeed::default();
|
||||
let old = manager
|
||||
.publish(&feed, Tracks::default())
|
||||
.expect("first publish");
|
||||
manager.outgoing.clear();
|
||||
manager.stop_publishing();
|
||||
let new = manager
|
||||
.publish(&feed, Tracks::default())
|
||||
.expect("next publish");
|
||||
for _ in 0..127 {
|
||||
manager.enqueue(
|
||||
ICE,
|
||||
&new,
|
||||
"viewer",
|
||||
"candidate:fixture",
|
||||
0,
|
||||
Tracks::default(),
|
||||
);
|
||||
}
|
||||
let stops: std::collections::HashSet<_> = manager
|
||||
.outgoing
|
||||
.iter()
|
||||
.filter(|control| control.action == STOP)
|
||||
.map(|control| control.share_id.clone())
|
||||
.collect();
|
||||
assert!(stops.contains(&old) && stops.contains(&new));
|
||||
assert!(!manager.publication_pending());
|
||||
}
|
||||
}
|
||||
|
|
@ -12,14 +12,41 @@ use std::{
|
|||
};
|
||||
|
||||
/// Latest owned preview frame and selected audio level; queues never grow.
|
||||
#[derive(Clone, Default)]
|
||||
#[derive(Clone)]
|
||||
pub struct CaptureFeed {
|
||||
video: Arc<Mutex<Option<VideoFrame>>>,
|
||||
video: Arc<Mutex<Option<Arc<VideoFrame>>>>,
|
||||
video_tx: Arc<tokio::sync::broadcast::Sender<Arc<VideoFrame>>>,
|
||||
audio_tx: Arc<tokio::sync::broadcast::Sender<Arc<super::AudioFrame>>>,
|
||||
audio: Arc<Mutex<f32>>,
|
||||
received_audio: Arc<AtomicBool>,
|
||||
}
|
||||
impl Default for CaptureFeed {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
video: Arc::default(),
|
||||
audio: Arc::default(),
|
||||
received_audio: Arc::default(),
|
||||
video_tx: Arc::new(tokio::sync::broadcast::channel(2).0),
|
||||
audio_tx: Arc::new(tokio::sync::broadcast::channel(8).0),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl CaptureFeed {
|
||||
pub fn take_frame(&self) -> Option<VideoFrame> {
|
||||
#[cfg(test)]
|
||||
pub(super) fn fixture_video(&self, frame: VideoFrame) {
|
||||
let _ = self.video_tx.send(Arc::new(frame));
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(super) fn fixture_audio(&self, frame: super::AudioFrame) {
|
||||
let _ = self.audio_tx.send(Arc::new(frame));
|
||||
}
|
||||
pub fn subscribe_video(&self) -> tokio::sync::broadcast::Receiver<Arc<VideoFrame>> {
|
||||
self.video_tx.subscribe()
|
||||
}
|
||||
pub fn subscribe_audio(&self) -> tokio::sync::broadcast::Receiver<Arc<super::AudioFrame>> {
|
||||
self.audio_tx.subscribe()
|
||||
}
|
||||
pub fn take_frame(&self) -> Option<Arc<VideoFrame>> {
|
||||
self.video.lock().ok()?.take()
|
||||
}
|
||||
pub fn audio_level(&self) -> f32 {
|
||||
|
|
@ -87,6 +114,7 @@ impl CaptureSession {
|
|||
source.set_property("path", portal.node.to_string());
|
||||
}
|
||||
strict_source(&source);
|
||||
source.set_property("keepalive-time", 500i32);
|
||||
let pipeline = video_pipeline(source, &session.feed, options)?;
|
||||
session.pipelines.push(pipeline);
|
||||
if let AudioChoice::Application(serial) | AudioChoice::System(serial) = options.audio {
|
||||
|
|
@ -177,11 +205,13 @@ fn video_pipeline(
|
|||
.sync(false)
|
||||
.build();
|
||||
let latest = feed.video.clone();
|
||||
let tx = feed.video_tx.clone();
|
||||
sink.set_callbacks(
|
||||
gstreamer_app::AppSinkCallbacks::builder()
|
||||
.new_sample(move |sink| {
|
||||
let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
|
||||
let frame = copy_video(&sample).map_err(|_| gst::FlowError::Error)?;
|
||||
let frame = Arc::new(copy_video(&sample).map_err(|_| gst::FlowError::Error)?);
|
||||
let _ = tx.send(frame.clone());
|
||||
*latest.lock().map_err(|_| gst::FlowError::Error)? = Some(frame);
|
||||
Ok(gst::FlowSuccess::Ok)
|
||||
})
|
||||
|
|
@ -214,7 +244,7 @@ fn video_pipeline(
|
|||
Ok(pipeline)
|
||||
}
|
||||
|
||||
fn copy_video(sample: &gst::Sample) -> Result<VideoFrame, String> {
|
||||
pub(super) fn copy_video(sample: &gst::Sample) -> Result<VideoFrame, String> {
|
||||
let caps = sample.caps().ok_or("Missing video caps")?;
|
||||
let info = gstreamer_video::VideoInfo::from_caps(caps).map_err(error)?;
|
||||
if info.format() != gstreamer_video::VideoFormat::Rgba
|
||||
|
|
@ -261,6 +291,7 @@ fn audio_pipeline(source: gst::Element, feed: &CaptureFeed) -> Result<gst::Pipel
|
|||
.build();
|
||||
let level = feed.audio.clone();
|
||||
let received = feed.received_audio.clone();
|
||||
let tx = feed.audio_tx.clone();
|
||||
sink.set_callbacks(
|
||||
gstreamer_app::AppSinkCallbacks::builder()
|
||||
.new_sample(move |sink| {
|
||||
|
|
@ -273,15 +304,22 @@ fn audio_pipeline(source: gst::Element, feed: &CaptureFeed) -> Result<gst::Pipel
|
|||
if map.len() > 48_000 * 8 {
|
||||
return Err(gst::FlowError::Error);
|
||||
}
|
||||
let mut samples = Vec::with_capacity(map.len() / 4);
|
||||
let mut sum = 0.0f64;
|
||||
let mut count = 0u32;
|
||||
for bytes in map.as_slice().as_chunks::<4>().0 {
|
||||
let value = f32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
|
||||
if value.is_finite() {
|
||||
samples.push(value);
|
||||
sum += f64::from(value) * f64::from(value);
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
let _ = tx.send(Arc::new(super::AudioFrame {
|
||||
samples,
|
||||
rate: 48_000,
|
||||
channels: 2,
|
||||
}));
|
||||
received.store(true, Ordering::Relaxed);
|
||||
*level.lock().map_err(|_| gst::FlowError::Error)? = if count == 0 {
|
||||
0.0
|
||||
|
|
@ -324,12 +362,12 @@ mod tests {
|
|||
#[test]
|
||||
fn silent_unavailable_source_times_out_and_drop_clears_preview() {
|
||||
let feed = CaptureFeed::default();
|
||||
*feed.video.lock().expect("frame lock") = Some(VideoFrame {
|
||||
*feed.video.lock().expect("frame lock") = Some(Arc::new(VideoFrame {
|
||||
width: 1,
|
||||
height: 1,
|
||||
rgba: vec![1, 2, 3, 255],
|
||||
timestamp_ns: None,
|
||||
});
|
||||
}));
|
||||
*feed.audio.lock().expect("level lock") = 0.5;
|
||||
let session = CaptureSession {
|
||||
pipelines: Vec::new(),
|
||||
|
|
|
|||
245
src/media/gateway_tests.rs
Normal file
245
src/media/gateway_tests.rs
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
//! Synthetic full native-client → authenticated gateway → peer media fixture.
|
||||
use super::{
|
||||
AudioFrame, CaptureFeed, VideoFrame,
|
||||
broadcast::BroadcastManager,
|
||||
peer::{AudioTrack, Tracks},
|
||||
};
|
||||
use crate::{
|
||||
identity,
|
||||
net::{self, NetCommand, NetEvent, NetHandle},
|
||||
};
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
process::{Child, Command, Stdio},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
struct Gateway {
|
||||
child: Child,
|
||||
directory: PathBuf,
|
||||
}
|
||||
impl Drop for Gateway {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
let _ = std::fs::remove_dir_all(&self.directory);
|
||||
}
|
||||
}
|
||||
async fn wait(handle: &NetHandle, test: impl Fn(&NetEvent) -> bool) -> NetEvent {
|
||||
tokio::time::timeout(Duration::from_secs(8), async {
|
||||
loop {
|
||||
if let Some(event) = handle.try_recv() {
|
||||
if let NetEvent::Disconnected { reason, .. } = &event {
|
||||
panic!("fixture disconnected: {reason}");
|
||||
}
|
||||
if let NetEvent::Error { message, .. } = &event {
|
||||
panic!("gateway rejected fixture: {message}");
|
||||
}
|
||||
if test(&event) {
|
||||
return event;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("authenticated gateway fixture timeout")
|
||||
}
|
||||
async fn connect(handle: &NetHandle, address: &str) -> String {
|
||||
handle
|
||||
.send(NetCommand::Connect {
|
||||
address: address.into(),
|
||||
auto_reconnect: false,
|
||||
})
|
||||
.await;
|
||||
let NetEvent::Connected { session_id, .. } =
|
||||
wait(handle, |event| matches!(event, NetEvent::Connected { .. })).await
|
||||
else {
|
||||
panic!("connected event")
|
||||
};
|
||||
handle
|
||||
.send(NetCommand::JoinChannel {
|
||||
channel_id: "voice".into(),
|
||||
})
|
||||
.await;
|
||||
wait(
|
||||
handle,
|
||||
|event| matches!(event, NetEvent::ChannelState { kind, .. } if kind=="voice"),
|
||||
)
|
||||
.await;
|
||||
// Browsing text must preserve the independently authorized voice membership.
|
||||
handle
|
||||
.send(NetCommand::JoinChannel {
|
||||
channel_id: "general".into(),
|
||||
})
|
||||
.await;
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
session_id
|
||||
}
|
||||
async fn pump(handle: &NetHandle, manager: &mut BroadcastManager) {
|
||||
while let Some(event) = handle.try_recv() {
|
||||
match event {
|
||||
NetEvent::MediaEvent(event) => manager.on_event(event).expect("authorized media event"),
|
||||
NetEvent::Error { message, .. } => panic!("gateway rejected media fixture: {message}"),
|
||||
NetEvent::Disconnected { reason, .. } => {
|
||||
panic!("gateway fixture disconnected: {reason}")
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
manager.tick();
|
||||
if let Some(error) = manager.take_error() {
|
||||
panic!("media fixture failed: {error}");
|
||||
}
|
||||
while let Some(control) = manager.take_control() {
|
||||
handle.send(NetCommand::MediaControl(control)).await;
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
#[ignore = "requires freshly built current VNOX_GATEWAY_BIN; uses synthetic media only"]
|
||||
async fn gateway_authorizes_real_two_peer_video_and_application_audio() {
|
||||
let binary = std::env::var_os("VNOX_GATEWAY_BIN").expect("set VNOX_GATEWAY_BIN");
|
||||
let directory =
|
||||
std::env::temp_dir().join(format!("vnox-media-gateway-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&directory).expect("fixture dir");
|
||||
let port = std::net::TcpListener::bind("127.0.0.1:0")
|
||||
.expect("port")
|
||||
.local_addr()
|
||||
.expect("address")
|
||||
.port();
|
||||
let voice_port = std::net::UdpSocket::bind("127.0.0.1:0")
|
||||
.expect("UDP port")
|
||||
.local_addr()
|
||||
.expect("UDP address")
|
||||
.port();
|
||||
let config = format!(
|
||||
r#"
|
||||
[node]
|
||||
name="media-fixture"
|
||||
address="127.0.0.1"
|
||||
[gateway]
|
||||
bind="127.0.0.1:{port}"
|
||||
admin_bind="127.0.0.1:0"
|
||||
message_rate_per_sec=0
|
||||
[voice]
|
||||
bind="127.0.0.1:{voice_port}"
|
||||
[storage]
|
||||
data_dir="{}"
|
||||
backend="sqlite"
|
||||
sqlite_path="{}/fixture.db"
|
||||
[server]
|
||||
mode="private"
|
||||
"#,
|
||||
directory.display(),
|
||||
directory.display()
|
||||
);
|
||||
let config_path = directory.join("config.toml");
|
||||
std::fs::write(&config_path, config).expect("config");
|
||||
let child = Command::new(binary)
|
||||
.arg("--config")
|
||||
.arg(config_path)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.expect("start actual gateway");
|
||||
let _gateway = Gateway { child, directory };
|
||||
tokio::time::sleep(Duration::from_millis(700)).await;
|
||||
let publisher_net = net::spawn(identity::generate("Synthetic publisher"));
|
||||
let viewer_net = net::spawn(identity::generate("Synthetic viewer"));
|
||||
let address = format!("127.0.0.1:{port}");
|
||||
let publisher_session = connect(&publisher_net, &address).await;
|
||||
let viewer_session = connect(&viewer_net, &address).await;
|
||||
let mut publisher = BroadcastManager::default();
|
||||
publisher.set_scope(&publisher_session, Some("voice"));
|
||||
let mut viewer = BroadcastManager::default();
|
||||
viewer.set_scope(&viewer_session, Some("voice"));
|
||||
let feed = CaptureFeed::default();
|
||||
let share = publisher
|
||||
.publish(
|
||||
&feed,
|
||||
Tracks {
|
||||
application_audio: true,
|
||||
microphone: false,
|
||||
},
|
||||
)
|
||||
.expect("explicit publish");
|
||||
let start = Instant::now();
|
||||
let mut samples = 0usize;
|
||||
let mut requested = false;
|
||||
let mut received_pcm = Vec::new();
|
||||
let mut received_video = false;
|
||||
while start.elapsed() < Duration::from_secs(15) {
|
||||
feed.fixture_video(VideoFrame {
|
||||
width: 64,
|
||||
height: 48,
|
||||
rgba: [200, 50, 70, 255].repeat(64 * 48),
|
||||
timestamp_ns: None,
|
||||
});
|
||||
let pcm = (samples..samples + 960)
|
||||
.flat_map(|n| {
|
||||
let sample = (n as f32 * 440.0 * std::f32::consts::TAU / 48_000.0).sin() * 0.3;
|
||||
[sample, sample]
|
||||
})
|
||||
.collect();
|
||||
samples += 960;
|
||||
feed.fixture_audio(AudioFrame {
|
||||
samples: pcm,
|
||||
rate: 48_000,
|
||||
channels: 2,
|
||||
});
|
||||
pump(&publisher_net, &mut publisher).await;
|
||||
pump(&viewer_net, &mut viewer).await;
|
||||
if !requested && viewer.shares().iter().any(|item| item.id == share) {
|
||||
viewer.watch(&share).expect("explicit watch");
|
||||
requested = true;
|
||||
}
|
||||
if let Some(remote) = viewer.remote_feed(&share) {
|
||||
if let Some(frame) = remote.take_video() {
|
||||
assert_eq!((frame.width, frame.height), (64, 48));
|
||||
received_video = true;
|
||||
}
|
||||
while let Some(pcm) = remote.take_audio(AudioTrack::Application) {
|
||||
received_pcm.extend(pcm.samples.into_iter().step_by(2));
|
||||
}
|
||||
assert_eq!(
|
||||
remote
|
||||
.microphone_samples
|
||||
.load(std::sync::atomic::Ordering::Relaxed),
|
||||
0,
|
||||
"unrequested microphone not delivered"
|
||||
);
|
||||
}
|
||||
if received_video && received_pcm.len() > 24_000 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert!(
|
||||
publisher.publishing() && publisher.viewer_count() == 1,
|
||||
"gateway authorized connected viewer"
|
||||
);
|
||||
assert!(
|
||||
received_video && received_pcm.len() > 24_000,
|
||||
"actual decoded video and Opus reached consenting viewer"
|
||||
);
|
||||
let frequency = received_pcm
|
||||
.windows(2)
|
||||
.filter(|p| p[0] < 0.0 && p[1] >= 0.0)
|
||||
.count() as f32
|
||||
* 48_000.0
|
||||
/ received_pcm.len() as f32;
|
||||
assert!(
|
||||
(frequency - 440.0).abs() < 60.0,
|
||||
"actual selected fixture audio: {frequency} Hz"
|
||||
);
|
||||
publisher.stop_publishing();
|
||||
pump(&publisher_net, &mut publisher).await;
|
||||
let stop = Instant::now();
|
||||
while viewer.is_watching(&share) && stop.elapsed() < Duration::from_secs(3) {
|
||||
pump(&viewer_net, &mut viewer).await;
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
assert!(
|
||||
!viewer.is_watching(&share),
|
||||
"authenticated STOP revoked receiver transport"
|
||||
);
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@
|
|||
mod policy;
|
||||
mod sources;
|
||||
|
||||
pub use policy::{AudioChoice, AudioKind, AudioSource, CaptureOptions, VideoFrame};
|
||||
pub use policy::{AudioChoice, AudioFrame, AudioKind, AudioSource, CaptureOptions, VideoFrame};
|
||||
pub use sources::{audio_sources, parse_audio_sources, validate_audio_choice};
|
||||
|
||||
#[cfg(all(target_os = "linux", feature = "screen-capture"))]
|
||||
|
|
@ -29,3 +29,11 @@ pub fn unavailable_reason() -> Option<&'static str> {
|
|||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(target_os = "linux", feature = "screen-capture"))]
|
||||
pub mod broadcast;
|
||||
#[cfg(all(target_os = "linux", feature = "screen-capture"))]
|
||||
pub mod peer;
|
||||
|
||||
#[cfg(all(test, target_os = "linux", feature = "screen-capture"))]
|
||||
mod gateway_tests;
|
||||
|
|
|
|||
646
src/media/peer.rs
Normal file
646
src/media/peer.rs
Normal file
|
|
@ -0,0 +1,646 @@
|
|||
//! Established ICE/DTLS-SRTP transport via GStreamer webrtcbin, with independent tracks.
|
||||
use super::{AudioFrame, VideoFrame};
|
||||
use gst::prelude::*;
|
||||
use gstreamer as gst;
|
||||
use gstreamer_webrtc as rtc;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
mpsc,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct Tracks {
|
||||
pub application_audio: bool,
|
||||
pub microphone: bool,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum Signal {
|
||||
Offer(String),
|
||||
Answer(String),
|
||||
Ice { mline: u32, candidate: String },
|
||||
Error(String),
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum AudioTrack {
|
||||
Application,
|
||||
Microphone,
|
||||
}
|
||||
#[derive(Clone, Default)]
|
||||
pub struct RemoteFeed {
|
||||
video: Arc<Mutex<Option<Arc<VideoFrame>>>>,
|
||||
app: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
|
||||
mic: Arc<Mutex<std::collections::VecDeque<AudioFrame>>>,
|
||||
pub video_frames: Arc<AtomicU64>,
|
||||
pub application_samples: Arc<AtomicU64>,
|
||||
pub microphone_samples: Arc<AtomicU64>,
|
||||
}
|
||||
impl RemoteFeed {
|
||||
pub fn take_video(&self) -> Option<Arc<VideoFrame>> {
|
||||
self.video.lock().ok()?.take()
|
||||
}
|
||||
pub fn take_audio(&self, track: AudioTrack) -> Option<AudioFrame> {
|
||||
match track {
|
||||
AudioTrack::Application => self.app.lock().ok()?.pop_front(),
|
||||
AudioTrack::Microphone => self.mic.lock().ok()?.pop_front(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit administrator-configured ICE services. Empty means host candidates only.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct IceServers {
|
||||
pub stun: Option<String>,
|
||||
pub turn: Option<String>,
|
||||
}
|
||||
impl IceServers {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.stun.as_ref().is_some_and(|uri| {
|
||||
!uri.starts_with("stun://")
|
||||
|| uri.len() > 2048
|
||||
|| uri.len() <= 7
|
||||
|| uri.chars().any(char::is_whitespace)
|
||||
}) || self.turn.as_ref().is_some_and(|uri| {
|
||||
!(uri.starts_with("turn://") || uri.starts_with("turns://"))
|
||||
|| uri.len() > 2048
|
||||
|| uri.len() <= 7
|
||||
|| uri.chars().any(char::is_whitespace)
|
||||
}) {
|
||||
return Err("Invalid explicitly configured ICE server URI".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// One directed publisher/viewer connection. The authenticated gateway routes its signals.
|
||||
pub struct Peer {
|
||||
pipeline: gst::Pipeline,
|
||||
rtc: gst::Element,
|
||||
video: Option<gstreamer_app::AppSrc>,
|
||||
app: Option<gstreamer_app::AppSrc>,
|
||||
mic: Option<gstreamer_app::AppSrc>,
|
||||
events: mpsc::Receiver<Signal>,
|
||||
tx: mpsc::SyncSender<Signal>,
|
||||
pub remote: RemoteFeed,
|
||||
remote_set: bool,
|
||||
started: std::time::Instant,
|
||||
pending_ice: Vec<(u32, String)>,
|
||||
app_time: Mutex<Option<u64>>,
|
||||
mic_time: Mutex<Option<u64>>,
|
||||
}
|
||||
|
||||
pub fn check_backend() -> Result<(), String> {
|
||||
gst::init().map_err(error)?;
|
||||
for plugin in [
|
||||
"webrtcbin",
|
||||
"nicesrc",
|
||||
"nicesink",
|
||||
"dtlssrtpenc",
|
||||
"dtlssrtpdec",
|
||||
"vp8enc",
|
||||
"vp8dec",
|
||||
"rtpvp8pay",
|
||||
"rtpvp8depay",
|
||||
"opusenc",
|
||||
"opusdec",
|
||||
"rtpopuspay",
|
||||
"rtpopusdepay",
|
||||
] {
|
||||
if gst::ElementFactory::find(plugin).is_none() {
|
||||
return Err(format!("Missing WebRTC plugin: {plugin}"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
impl Peer {
|
||||
pub fn publisher(tracks: Tracks) -> Result<Self, String> {
|
||||
Self::new(true, tracks, &IceServers::default())
|
||||
}
|
||||
pub fn viewer(tracks: Tracks) -> Result<Self, String> {
|
||||
Self::new(false, tracks, &IceServers::default())
|
||||
}
|
||||
pub fn publisher_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
|
||||
Self::new(true, tracks, ice)
|
||||
}
|
||||
pub fn viewer_with_ice(tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
|
||||
Self::new(false, tracks, ice)
|
||||
}
|
||||
fn new(publisher: bool, tracks: Tracks, ice: &IceServers) -> Result<Self, String> {
|
||||
ice.validate()?;
|
||||
check_backend()?;
|
||||
let pipeline = gst::Pipeline::new();
|
||||
let rtc = gst::ElementFactory::make("webrtcbin")
|
||||
.name("transport")
|
||||
.build()
|
||||
.map_err(error)?;
|
||||
rtc.set_property_from_str("bundle-policy", "max-bundle");
|
||||
rtc.set_property("latency", 100u32);
|
||||
if let Some(uri) = &ice.stun {
|
||||
rtc.set_property("stun-server", uri);
|
||||
}
|
||||
if let Some(uri) = &ice.turn {
|
||||
rtc.set_property("turn-server", uri);
|
||||
}
|
||||
pipeline.add(&rtc).map_err(error)?;
|
||||
let (tx, events) = mpsc::sync_channel(64);
|
||||
let ice_tx = tx.clone();
|
||||
rtc.connect("on-ice-candidate", false, move |values| {
|
||||
if let (Ok(mline), Ok(candidate)) = (values[1].get::<u32>(), values[2].get::<String>())
|
||||
{
|
||||
let _ = ice_tx.try_send(Signal::Ice { mline, candidate });
|
||||
}
|
||||
None
|
||||
});
|
||||
let remote = RemoteFeed::default();
|
||||
if !publisher {
|
||||
let weak_pipeline = pipeline.downgrade();
|
||||
let feed = remote.clone();
|
||||
let receive_tx = tx.clone();
|
||||
rtc.connect_pad_added(move |_, pad| {
|
||||
if pad.direction() != gst::PadDirection::Src {
|
||||
return;
|
||||
}
|
||||
if let Some(pipeline) = weak_pipeline.upgrade()
|
||||
&& let Err(error) = receive(&pipeline, pad, &feed, tracks)
|
||||
{
|
||||
let _ = receive_tx.try_send(Signal::Error(error));
|
||||
}
|
||||
});
|
||||
}
|
||||
let video = if publisher {
|
||||
Some(add_sender(&pipeline, &rtc, 96, "screen-video", true)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let app = if publisher && tracks.application_audio {
|
||||
Some(add_sender(&pipeline, &rtc, 97, "application-audio", false)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mic = if publisher && tracks.microphone {
|
||||
Some(add_sender(&pipeline, &rtc, 98, "microphone", false)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
pipeline.set_state(gst::State::Playing).map_err(error)?;
|
||||
Ok(Self {
|
||||
pipeline,
|
||||
rtc,
|
||||
video,
|
||||
app,
|
||||
mic,
|
||||
events,
|
||||
tx,
|
||||
remote,
|
||||
remote_set: false,
|
||||
started: std::time::Instant::now(),
|
||||
pending_ice: Vec::new(),
|
||||
app_time: Mutex::new(None),
|
||||
mic_time: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
/// Create an SDP offer only after the gateway authorizes a viewer subscription.
|
||||
pub fn offer(&self) {
|
||||
create_description(&self.rtc, true, self.signal_sender());
|
||||
}
|
||||
fn signal_sender(&self) -> mpsc::SyncSender<Signal> {
|
||||
self.tx.clone()
|
||||
}
|
||||
pub fn apply_offer(&mut self, sdp: &str) -> Result<(), String> {
|
||||
self.set_description(sdp, true)?;
|
||||
create_description(&self.rtc, false, self.signal_sender());
|
||||
Ok(())
|
||||
}
|
||||
pub fn apply_answer(&mut self, sdp: &str) -> Result<(), String> {
|
||||
self.set_description(sdp, false)
|
||||
}
|
||||
fn set_description(&mut self, text: &str, offer: bool) -> Result<(), String> {
|
||||
if text.len() > 65_536 {
|
||||
return Err("SDP exceeds limit".into());
|
||||
}
|
||||
let sdp = gstreamer_sdp::SDPMessage::parse_buffer(text.as_bytes()).map_err(error)?;
|
||||
let description = rtc::WebRTCSessionDescription::new(
|
||||
if offer {
|
||||
rtc::WebRTCSDPType::Offer
|
||||
} else {
|
||||
rtc::WebRTCSDPType::Answer
|
||||
},
|
||||
sdp,
|
||||
);
|
||||
let promise = gst::Promise::new();
|
||||
self.rtc
|
||||
.emit_by_name::<()>("set-remote-description", &[&description, &promise]);
|
||||
if promise.wait() != gst::PromiseResult::Replied {
|
||||
return Err("Remote SDP was rejected".into());
|
||||
}
|
||||
if let Some(reply) = promise.get_reply()
|
||||
&& reply.has_field("error")
|
||||
{
|
||||
return Err("Remote SDP was rejected".into());
|
||||
}
|
||||
self.remote_set = true;
|
||||
for (mline, candidate) in self.pending_ice.drain(..) {
|
||||
self.rtc
|
||||
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn add_ice(&mut self, mline: u32, candidate: &str) -> Result<(), String> {
|
||||
if mline > 2 || candidate.len() > 2048 || self.pending_ice.len() >= 64 {
|
||||
return Err("ICE candidate exceeds limit".into());
|
||||
}
|
||||
if self.remote_set {
|
||||
self.rtc
|
||||
.emit_by_name::<()>("add-ice-candidate", &[&mline, &candidate]);
|
||||
} else {
|
||||
self.pending_ice.push((mline, candidate.into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn poll(&self) -> Result<Option<Signal>, String> {
|
||||
if let Some(bus) = self.pipeline.bus() {
|
||||
while let Some(message) = bus.pop() {
|
||||
match message.view() {
|
||||
gst::MessageView::Error(e) => return Err(format!("WebRTC: {}", e.error())),
|
||||
gst::MessageView::Eos(..) => return Err("WebRTC stream closed".into()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let state = self
|
||||
.rtc
|
||||
.property::<rtc::WebRTCPeerConnectionState>("connection-state");
|
||||
if self.started.elapsed() > std::time::Duration::from_secs(20)
|
||||
&& state != rtc::WebRTCPeerConnectionState::Connected
|
||||
{
|
||||
return Err("WebRTC connection timed out".into());
|
||||
}
|
||||
if matches!(
|
||||
state,
|
||||
rtc::WebRTCPeerConnectionState::Failed | rtc::WebRTCPeerConnectionState::Closed
|
||||
) {
|
||||
return Err("WebRTC peer disconnected".into());
|
||||
}
|
||||
Ok(self.events.try_recv().ok())
|
||||
}
|
||||
pub fn connected(&self) -> bool {
|
||||
self.rtc
|
||||
.property::<rtc::WebRTCPeerConnectionState>("connection-state")
|
||||
== rtc::WebRTCPeerConnectionState::Connected
|
||||
}
|
||||
pub fn push_video(&self, frame: &VideoFrame) -> Result<(), String> {
|
||||
frame.validate()?;
|
||||
let source = self.video.as_ref().ok_or("Peer does not publish video")?;
|
||||
source.set_caps(Some(
|
||||
&gst::Caps::builder("video/x-raw")
|
||||
.field("format", "RGBA")
|
||||
.field("width", frame.width as i32)
|
||||
.field("height", frame.height as i32)
|
||||
.field("framerate", gst::Fraction::new(30, 1))
|
||||
.build(),
|
||||
));
|
||||
source
|
||||
.push_buffer(gst::Buffer::from_mut_slice(frame.rgba.clone()))
|
||||
.map_err(error)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn push_audio(&self, track: AudioTrack, frame: &AudioFrame) -> Result<(), String> {
|
||||
frame.validate()?;
|
||||
let source = match track {
|
||||
AudioTrack::Application => &self.app,
|
||||
AudioTrack::Microphone => &self.mic,
|
||||
}
|
||||
.as_ref()
|
||||
.ok_or("Audio track was not explicitly enabled")?;
|
||||
source.set_caps(Some(
|
||||
&gst::Caps::builder("audio/x-raw")
|
||||
.field("format", "F32LE")
|
||||
.field("layout", "interleaved")
|
||||
.field("rate", frame.rate as i32)
|
||||
.field("channels", i32::from(frame.channels))
|
||||
.build(),
|
||||
));
|
||||
let bytes: Vec<u8> = frame
|
||||
.samples
|
||||
.iter()
|
||||
.flat_map(|sample| sample.to_le_bytes())
|
||||
.collect();
|
||||
let clock = match track {
|
||||
AudioTrack::Application => &self.app_time,
|
||||
AudioTrack::Microphone => &self.mic_time,
|
||||
};
|
||||
let mut next = clock
|
||||
.lock()
|
||||
.map_err(|_| "Audio timestamp clock unavailable")?;
|
||||
let pts = next.unwrap_or_else(|| {
|
||||
self.pipeline
|
||||
.current_running_time()
|
||||
.map(|time| time.nseconds())
|
||||
.unwrap_or(0)
|
||||
});
|
||||
let duration = (frame.samples.len() as u64 / u64::from(frame.channels)) * 1_000_000_000
|
||||
/ u64::from(frame.rate);
|
||||
let mut buffer = gst::Buffer::from_mut_slice(bytes);
|
||||
if let Some(buffer) = buffer.get_mut() {
|
||||
buffer.set_pts(gst::ClockTime::from_nseconds(pts));
|
||||
buffer.set_duration(gst::ClockTime::from_nseconds(duration));
|
||||
}
|
||||
*next = Some(pts.saturating_add(duration));
|
||||
source.push_buffer(buffer).map_err(error)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
impl Drop for Peer {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.pipeline.set_state(gst::State::Null);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_sender(
|
||||
pipeline: &gst::Pipeline,
|
||||
rtc: &gst::Element,
|
||||
payload: i32,
|
||||
msid: &str,
|
||||
video: bool,
|
||||
) -> Result<gstreamer_app::AppSrc, String> {
|
||||
let description = if video {
|
||||
format!(
|
||||
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=8388608 leaky-type=downstream ! queue max-size-buffers=2 leaky=downstream ! videoconvert ! vp8enc deadline=1 cpu-used=8 target-bitrate=1500000 keyframe-max-dist=60 ! rtpvp8pay pt={payload} ! capsfilter caps=\"application/x-rtp,media=video,encoding-name=VP8,payload={payload},clock-rate=90000\""
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"appsrc name=source is-live=true format=time do-timestamp=true block=false max-bytes=384000 leaky-type=downstream ! queue max-size-buffers=8 leaky=downstream ! audioconvert ! audioresample ! opusenc bitrate=64000 ! rtpopuspay pt={payload} ! capsfilter caps=\"application/x-rtp,media=audio,encoding-name=OPUS,payload={payload},clock-rate=48000\""
|
||||
)
|
||||
};
|
||||
let bin = gst::parse::bin_from_description(&description, true).map_err(error)?;
|
||||
let source = bin
|
||||
.by_name("source")
|
||||
.ok_or("Missing media appsrc")?
|
||||
.downcast::<gstreamer_app::AppSrc>()
|
||||
.map_err(|_| "Wrong media source type")?;
|
||||
pipeline.add(&bin).map_err(error)?;
|
||||
let src = bin.static_pad("src").ok_or("Missing RTP source pad")?;
|
||||
let sink = rtc
|
||||
.request_pad_simple("sink_%u")
|
||||
.ok_or("WebRTC refused a media track")?;
|
||||
sink.set_property("msid", msid);
|
||||
let transceiver = sink.property::<rtc::WebRTCRTPTransceiver>("transceiver");
|
||||
transceiver.set_property("direction", rtc::WebRTCRTPTransceiverDirection::Sendonly);
|
||||
src.link(&sink).map_err(error)?;
|
||||
Ok(source)
|
||||
}
|
||||
|
||||
fn create_description(webrtc: &gst::Element, offer: bool, tx: mpsc::SyncSender<Signal>) {
|
||||
let weak = webrtc.downgrade();
|
||||
let promise = gst::Promise::with_change_func(move |reply| {
|
||||
let result = (|| {
|
||||
let reply = reply
|
||||
.map_err(|e| format!("SDP promise failed: {e:?}"))?
|
||||
.ok_or("No SDP reply")?;
|
||||
let description = reply
|
||||
.get::<rtc::WebRTCSessionDescription>(if offer { "offer" } else { "answer" })
|
||||
.map_err(error)?;
|
||||
let webrtc = weak.upgrade().ok_or("WebRTC peer closed")?;
|
||||
webrtc.emit_by_name::<()>(
|
||||
"set-local-description",
|
||||
&[&description, &None::<gst::Promise>],
|
||||
);
|
||||
let text = description.sdp().as_text().map_err(error)?;
|
||||
Ok::<_, String>(if offer {
|
||||
Signal::Offer(text)
|
||||
} else {
|
||||
Signal::Answer(text)
|
||||
})
|
||||
})();
|
||||
let _ = tx.try_send(result.unwrap_or_else(Signal::Error));
|
||||
});
|
||||
webrtc.emit_by_name::<()>(
|
||||
if offer {
|
||||
"create-offer"
|
||||
} else {
|
||||
"create-answer"
|
||||
},
|
||||
&[&None::<gst::Structure>, &promise],
|
||||
);
|
||||
}
|
||||
|
||||
fn receive(
|
||||
pipeline: &gst::Pipeline,
|
||||
pad: &gst::Pad,
|
||||
feed: &RemoteFeed,
|
||||
tracks: Tracks,
|
||||
) -> Result<(), String> {
|
||||
let caps = pad.current_caps().ok_or("Incoming track has no caps")?;
|
||||
let payload = caps
|
||||
.structure(0)
|
||||
.ok_or("Incoming track has no format")?
|
||||
.get::<i32>("payload")
|
||||
.map_err(error)?;
|
||||
let video = payload == 96;
|
||||
if !(video || payload == 97 && tracks.application_audio || payload == 98 && tracks.microphone) {
|
||||
return Err("Unannounced remote track rejected".into());
|
||||
}
|
||||
let description = if video {
|
||||
"queue max-size-buffers=2 leaky=downstream ! rtpvp8depay ! vp8dec ! videoconvert ! video/x-raw,format=RGBA,width=[1,1920],height=[1,1080] ! appsink name=sink sync=false max-buffers=1 drop=true"
|
||||
} else {
|
||||
"queue max-size-buffers=8 leaky=downstream ! rtpopusdepay ! opusdec ! audioconvert ! audioresample ! audio/x-raw,format=F32LE,layout=interleaved,rate=48000,channels=2 ! appsink name=sink sync=false max-buffers=4 drop=true"
|
||||
};
|
||||
let bin = gst::parse::bin_from_description(description, true).map_err(error)?;
|
||||
let sink = bin
|
||||
.by_name("sink")
|
||||
.ok_or("Remote sink missing")?
|
||||
.downcast::<gstreamer_app::AppSink>()
|
||||
.map_err(|_| "Wrong remote sink")?;
|
||||
let feed = feed.clone();
|
||||
sink.set_callbacks(
|
||||
gstreamer_app::AppSinkCallbacks::builder()
|
||||
.new_sample(move |sink| {
|
||||
let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
|
||||
if video {
|
||||
let frame =
|
||||
super::capture::copy_video(&sample).map_err(|_| gst::FlowError::Error)?;
|
||||
*feed.video.lock().map_err(|_| gst::FlowError::Error)? = Some(Arc::new(frame));
|
||||
feed.video_frames.fetch_add(1, Ordering::Relaxed);
|
||||
} else {
|
||||
let map = sample
|
||||
.buffer()
|
||||
.ok_or(gst::FlowError::Error)?
|
||||
.map_readable()
|
||||
.map_err(|_| gst::FlowError::Error)?;
|
||||
if map.len() > 384000 {
|
||||
return Err(gst::FlowError::Error);
|
||||
}
|
||||
let samples: Vec<f32> = map
|
||||
.as_slice()
|
||||
.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|bytes| f32::from_le_bytes(*bytes))
|
||||
.collect();
|
||||
let count = samples.len() as u64;
|
||||
let frame = AudioFrame {
|
||||
samples,
|
||||
rate: 48000,
|
||||
channels: 2,
|
||||
};
|
||||
frame.validate().map_err(|_| gst::FlowError::Error)?;
|
||||
let (queue, received) = if payload == 97 {
|
||||
(&feed.app, &feed.application_samples)
|
||||
} else {
|
||||
(&feed.mic, &feed.microphone_samples)
|
||||
};
|
||||
let mut queue = queue.lock().map_err(|_| gst::FlowError::Error)?;
|
||||
if queue.len() == 8 {
|
||||
queue.pop_front();
|
||||
}
|
||||
queue.push_back(frame);
|
||||
received.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
Ok(gst::FlowSuccess::Ok)
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
pipeline.add(&bin).map_err(error)?;
|
||||
bin.sync_state_with_parent().map_err(error)?;
|
||||
pad.link(&bin.static_pad("sink").ok_or("Missing remote RTP sink")?)
|
||||
.map_err(error)?;
|
||||
Ok(())
|
||||
}
|
||||
fn error(error: impl std::fmt::Display) -> String {
|
||||
error.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::{Duration, Instant};
|
||||
fn forward(from: &Peer, to: &mut Peer) {
|
||||
while let Some(signal) = from.poll().expect("healthy peer") {
|
||||
match signal {
|
||||
Signal::Offer(sdp) => to.apply_offer(&sdp).expect("offer accepted"),
|
||||
Signal::Answer(sdp) => to.apply_answer(&sdp).expect("answer accepted"),
|
||||
Signal::Ice { mline, candidate } => {
|
||||
to.add_ice(mline, &candidate).expect("candidate accepted")
|
||||
}
|
||||
Signal::Error(error) => panic!("signalling failed: {error}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn two_peers_deliver_video_and_separate_application_and_microphone_tracks() {
|
||||
let tracks = Tracks {
|
||||
application_audio: true,
|
||||
microphone: true,
|
||||
};
|
||||
let mut publisher = Peer::publisher(tracks).expect("publisher");
|
||||
let mut viewer = Peer::viewer(tracks).expect("viewer");
|
||||
let frame = VideoFrame {
|
||||
width: 64,
|
||||
height: 48,
|
||||
rgba: [200, 50, 70, 255].repeat(64 * 48),
|
||||
timestamp_ns: None,
|
||||
};
|
||||
publisher.push_video(&frame).expect("fixture video");
|
||||
publisher.offer();
|
||||
let start = Instant::now();
|
||||
let mut index = 0usize;
|
||||
let mut app_pcm = Vec::new();
|
||||
let mut mic_pcm = Vec::new();
|
||||
while start.elapsed() < Duration::from_secs(12) {
|
||||
forward(&publisher, &mut viewer);
|
||||
forward(&viewer, &mut publisher);
|
||||
publisher.push_video(&frame).expect("video push");
|
||||
for (track, frequency) in [
|
||||
(AudioTrack::Application, 440.0f32),
|
||||
(AudioTrack::Microphone, 880.0f32),
|
||||
] {
|
||||
let samples = (index..index + 960)
|
||||
.flat_map(|sample| {
|
||||
let value = (sample as f32 * frequency * std::f32::consts::TAU / 48_000.0)
|
||||
.sin()
|
||||
* 0.3;
|
||||
[value, value]
|
||||
})
|
||||
.collect();
|
||||
publisher
|
||||
.push_audio(
|
||||
track,
|
||||
&AudioFrame {
|
||||
samples,
|
||||
rate: 48_000,
|
||||
channels: 2,
|
||||
},
|
||||
)
|
||||
.expect("independent audio push");
|
||||
}
|
||||
index += 960;
|
||||
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Application) {
|
||||
app_pcm.extend(frame.samples.into_iter().step_by(2));
|
||||
}
|
||||
while let Some(frame) = viewer.remote.take_audio(AudioTrack::Microphone) {
|
||||
mic_pcm.extend(frame.samples.into_iter().step_by(2));
|
||||
}
|
||||
if viewer.remote.video_frames.load(Ordering::Relaxed) >= 3
|
||||
&& app_pcm.len() > 24_000
|
||||
&& mic_pcm.len() > 24_000
|
||||
{
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
assert!(
|
||||
publisher.connected() && viewer.connected(),
|
||||
"DTLS/ICE connection established"
|
||||
);
|
||||
let received = viewer
|
||||
.remote
|
||||
.take_video()
|
||||
.expect("actual decoded remote video");
|
||||
assert_eq!((received.width, received.height), (64, 48));
|
||||
assert!((i32::from(received.rgba[0]) - 200).abs() < 20);
|
||||
assert!(
|
||||
app_pcm.len() > 24_000 && mic_pcm.len() > 24_000,
|
||||
"both distinct Opus tracks delivered"
|
||||
);
|
||||
let frequency = |pcm: &[f32]| {
|
||||
pcm.windows(2)
|
||||
.filter(|pair| pair[0] < 0.0 && pair[1] >= 0.0)
|
||||
.count() as f32
|
||||
* 48_000.0
|
||||
/ pcm.len() as f32
|
||||
};
|
||||
let app_frequency = frequency(&app_pcm);
|
||||
let mic_frequency = frequency(&mic_pcm);
|
||||
assert!(
|
||||
(app_frequency - 440.0).abs() < 60.0,
|
||||
"app frequency {app_frequency}"
|
||||
);
|
||||
assert!(
|
||||
(mic_frequency - 880.0).abs() < 100.0,
|
||||
"microphone frequency {mic_frequency}"
|
||||
);
|
||||
assert!(
|
||||
publisher
|
||||
.push_audio(
|
||||
AudioTrack::Application,
|
||||
&AudioFrame {
|
||||
samples: vec![f32::NAN],
|
||||
rate: 48_000,
|
||||
channels: 1
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn unrequested_audio_track_cannot_be_enabled_by_push() {
|
||||
let publisher = Peer::publisher(Tracks::default()).expect("publisher");
|
||||
let pcm = AudioFrame {
|
||||
samples: vec![0.0; 960],
|
||||
rate: 48_000,
|
||||
channels: 1,
|
||||
};
|
||||
assert!(publisher.push_audio(AudioTrack::Microphone, &pcm).is_err());
|
||||
assert!(publisher.push_audio(AudioTrack::Application, &pcm).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -107,3 +107,28 @@ mod tests {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One bounded independent PCM source, ready for an Opus track.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AudioFrame {
|
||||
pub samples: Vec<f32>,
|
||||
pub rate: u32,
|
||||
pub channels: u16,
|
||||
}
|
||||
impl AudioFrame {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.rate != 48_000
|
||||
|| !(1..=2).contains(&self.channels)
|
||||
|| self.samples.len() > 48_000 * 2
|
||||
|| self.samples.is_empty()
|
||||
|| !self
|
||||
.samples
|
||||
.len()
|
||||
.is_multiple_of(usize::from(self.channels))
|
||||
|| self.samples.iter().any(|sample| !sample.is_finite())
|
||||
{
|
||||
return Err("Invalid or unbounded PCM frame".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue