diff --git a/src/audio/config.rs b/src/audio/config.rs index 304dd5f..0b4524b 100644 --- a/src/audio/config.rs +++ b/src/audio/config.rs @@ -1,6 +1,6 @@ use anyhow::{Result, anyhow}; use cpal::traits::DeviceTrait; -use cpal::{SampleFormat, StreamConfig}; +use cpal::{SampleFormat, StreamConfig, SupportedStreamConfigRange}; use tracing::info; pub const OPUS_RATE: u32 = 48_000; @@ -41,24 +41,19 @@ impl AudioConfig { device.supported_output_configs()?.collect() }; - // Prefer 48 kHz mono F32 — matches Opus native rate. - for r in &ranges { - if r.channels() == 1 - && r.sample_format() == SampleFormat::F32 - && r.min_sample_rate() <= OPUS_RATE - && r.max_sample_rate() >= OPUS_RATE - { - let stream = r.with_sample_rate(OPUS_RATE).config(); - info!( - "audio {}: {} Hz mono", - if is_input { "in" } else { "out" }, - OPUS_RATE - ); - return Ok(Self { - stream, - sample_rate: OPUS_RATE, - }); - } + // Streams are built with f32 callbacks, so only f32 configurations are usable. + // Prefer 48 kHz (Opus native rate) with the fewest channels, mono first. + let direction = if is_input { "in" } else { "out" }; + if let Some(range) = best_opus_range(&ranges) { + let stream = range.with_sample_rate(OPUS_RATE).config(); + info!( + "audio {direction}: {OPUS_RATE} Hz {}ch f32", + stream.channels + ); + return Ok(Self { + stream, + sample_rate: OPUS_RATE, + }); } // Fallback: device native rate, resample to/from Opus in capture/playback. @@ -68,14 +63,20 @@ impl AudioConfig { device.default_output_config() } .map_err(|e| anyhow!("no default audio config: {e}"))?; + if default.sample_format() != SampleFormat::F32 { + return Err(anyhow!( + "the {} device offers {:?} samples but 32-bit float is required; \ + pick another device in the audio settings", + if is_input { "input" } else { "output" }, + default.sample_format() + )); + } let sample_rate = default.sample_rate(); let stream: StreamConfig = default.config(); info!( - "audio {}: {} Hz {}ch (will resample for Opus)", - if is_input { "in" } else { "out" }, - sample_rate, - stream.channels + "audio {direction}: {} Hz {}ch (will resample for Opus)", + sample_rate, stream.channels ); Ok(Self { stream, @@ -84,6 +85,18 @@ impl AudioConfig { } } +/// Picks the f32 range that covers 48 kHz with the fewest channels (mono preferred). +fn best_opus_range(ranges: &[SupportedStreamConfigRange]) -> Option<&SupportedStreamConfigRange> { + ranges + .iter() + .filter(|r| { + r.sample_format() == SampleFormat::F32 + && r.min_sample_rate() <= OPUS_RATE + && r.max_sample_rate() >= OPUS_RATE + }) + .min_by_key(|r| (r.channels() != 1, r.channels())) +} + pub fn frame_samples(sample_rate: u32) -> usize { (sample_rate as usize * FRAME_MS as usize) / 1000 } @@ -118,7 +131,47 @@ pub fn resample_rate(input: &[f32], from_hz: u32, to_hz: u32) -> Vec { #[cfg(test)] mod tests { - use super::{bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting}; + use super::{ + OPUS_RATE, SampleFormat, SupportedStreamConfigRange, best_opus_range, + bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting, + }; + + use cpal::SupportedBufferSize; + + fn range( + channels: u16, + min: u32, + max: u32, + format: SampleFormat, + ) -> SupportedStreamConfigRange { + SupportedStreamConfigRange::new(channels, min, max, SupportedBufferSize::Unknown, format) + } + + #[test] + fn config_prefers_mono_f32_at_the_opus_rate() { + let ranges = [ + range(2, 1, 384_000, SampleFormat::F32), + range(1, 1, 384_000, SampleFormat::I16), + range(1, 1, 384_000, SampleFormat::F32), + ]; + let best = best_opus_range(&ranges).expect("f32 range"); + assert_eq!( + (best.channels(), best.sample_format()), + (1, SampleFormat::F32) + ); + } + + #[test] + fn config_falls_back_to_stereo_and_rejects_non_f32_or_wrong_rate() { + let stereo = [range(2, 8_000, 96_000, SampleFormat::F32)]; + assert_eq!(best_opus_range(&stereo).map(|r| r.channels()), Some(2)); + let unusable = [ + range(1, 8_000, 44_100, SampleFormat::F32), + range(1, 1, 96_000, SampleFormat::I16), + ]; + assert!(best_opus_range(&unusable).is_none()); + assert_eq!(OPUS_RATE, 48_000); + } #[test] fn jitter_size_maps_to_ms() { diff --git a/src/audio/diagnostics.rs b/src/audio/diagnostics.rs new file mode 100644 index 0000000..aeb0f96 --- /dev/null +++ b/src/audio/diagnostics.rs @@ -0,0 +1,201 @@ +//! Shared audio health: failures raised on capture and playback threads must reach the UI. +//! +//! Earlier versions only logged such failures (`tracing::error!` on a detached thread), so a +//! microphone that could not be opened still showed a connected voice room with no sound. The +//! threads now report into [`Diagnostics`]; the UI polls it and shows the message. + +use std::sync::{ + Mutex, + atomic::{AtomicU32, AtomicU64, Ordering}, +}; + +/// Which half of the audio pipeline raised an issue. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioStage { + Capture, + Playback, +} + +/// One reportable audio problem. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioIssue { + pub stage: AudioStage, + pub message: String, + /// The stage cannot continue (stream not built, thread ended). + pub fatal: bool, +} + +/// Counters for support logs: where in the chain did the signal stop? +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct AudioStats { + pub captured_frames: u64, + pub transmitted_frames: u64, + pub received_packets: u64, + pub decoded_frames: u64, + /// Loudest input sample seen since the last snapshot, 0.0 to 1.0. + pub input_peak: f32, +} + +#[derive(Default)] +struct Failures { + capture: bool, + playback: bool, +} + +/// Thread-safe sink for issues and counters, shared by the pipeline threads. +#[derive(Default)] +pub struct Diagnostics { + issues: Mutex>, + failed: Mutex, + last_message: Mutex>, + captured_frames: AtomicU64, + transmitted_frames: AtomicU64, + received_packets: AtomicU64, + decoded_frames: AtomicU64, + input_peak_bits: AtomicU32, +} + +impl Diagnostics { + /// Records an issue (and logs it). Identical consecutive reports from one stage are + /// collapsed so a failing device callback cannot flood the UI. + pub fn report(&self, stage: AudioStage, fatal: bool, message: impl Into) { + let message = message.into(); + if fatal && let Ok(mut failed) = self.failed.lock() { + match stage { + AudioStage::Capture => failed.capture = true, + AudioStage::Playback => failed.playback = true, + } + } + if let Ok(mut last) = self.last_message.lock() { + if last + .as_ref() + .is_some_and(|(s, m)| *s == stage && *m == message) + { + return; + } + *last = Some((stage, message.clone())); + } + tracing::error!(?stage, fatal, "audio: {message}"); + if let Ok(mut issues) = self.issues.lock() { + if issues.len() >= 16 { + issues.remove(0); + } + issues.push(AudioIssue { + stage, + message, + fatal, + }); + } + } + + /// Drains the issues reported since the last call. + pub fn take_issues(&self) -> Vec { + self.issues + .lock() + .map(|mut issues| std::mem::take(&mut *issues)) + .unwrap_or_default() + } + + /// True when neither capture nor playback is working any more. + pub fn all_failed(&self) -> bool { + self.failed + .lock() + .map(|failed| failed.capture && failed.playback) + .unwrap_or(false) + } + + pub fn count_captured(&self, peak: f32) { + self.captured_frames.fetch_add(1, Ordering::Relaxed); + let mut current = self.input_peak_bits.load(Ordering::Relaxed); + while peak > f32::from_bits(current) { + match self.input_peak_bits.compare_exchange_weak( + current, + peak.to_bits(), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + + pub fn count_transmitted(&self) { + self.transmitted_frames.fetch_add(1, Ordering::Relaxed); + } + + pub fn count_received(&self) { + self.received_packets.fetch_add(1, Ordering::Relaxed); + } + + pub fn count_decoded(&self) { + self.decoded_frames.fetch_add(1, Ordering::Relaxed); + } + + /// Current counters; the input peak is reset so each snapshot covers one interval. + pub fn snapshot(&self) -> AudioStats { + AudioStats { + captured_frames: self.captured_frames.load(Ordering::Relaxed), + transmitted_frames: self.transmitted_frames.load(Ordering::Relaxed), + received_packets: self.received_packets.load(Ordering::Relaxed), + decoded_frames: self.decoded_frames.load(Ordering::Relaxed), + input_peak: f32::from_bits(self.input_peak_bits.swap(0, Ordering::Relaxed)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn issues_are_delivered_once_and_duplicates_collapse() { + let diag = Diagnostics::default(); + diag.report(AudioStage::Capture, true, "no microphone"); + diag.report(AudioStage::Capture, true, "no microphone"); + diag.report(AudioStage::Playback, false, "xrun"); + let issues = diag.take_issues(); + assert_eq!(issues.len(), 2); + assert_eq!(issues[0].stage, AudioStage::Capture); + assert!(issues[0].fatal); + assert!(diag.take_issues().is_empty()); + } + + #[test] + fn all_failed_requires_both_stages_to_fail_fatally() { + let diag = Diagnostics::default(); + diag.report(AudioStage::Capture, true, "a"); + assert!(!diag.all_failed()); + diag.report(AudioStage::Playback, false, "b"); + assert!(!diag.all_failed()); + diag.report(AudioStage::Playback, true, "c"); + assert!(diag.all_failed()); + } + + #[test] + fn snapshot_reports_counters_and_resets_the_peak() { + let diag = Diagnostics::default(); + diag.count_captured(0.2); + diag.count_captured(0.5); + diag.count_captured(0.1); + diag.count_transmitted(); + diag.count_received(); + diag.count_decoded(); + let first = diag.snapshot(); + assert_eq!(first.captured_frames, 3); + assert_eq!(first.transmitted_frames, 1); + assert_eq!(first.received_packets, 1); + assert_eq!(first.decoded_frames, 1); + assert!((first.input_peak - 0.5).abs() < f32::EPSILON); + assert_eq!(diag.snapshot().input_peak, 0.0); + } + + #[test] + fn issue_queue_is_bounded() { + let diag = Diagnostics::default(); + for i in 0..40 { + diag.report(AudioStage::Playback, false, format!("e{i}")); + } + assert_eq!(diag.take_issues().len(), 16); + } +} diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 0b9b3a8..97b9b4e 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -1,12 +1,14 @@ pub mod config; pub mod devices; +pub mod diagnostics; mod pipeline; mod processing; pub use devices::{DeviceEntry, DeviceLists}; +pub use diagnostics::{AudioIssue, AudioStage, AudioStats}; pub use pipeline::JitterConfig; pub use pipeline::{AudioPipeline, MediaMixer}; -pub use processing::{DenoiseState, TransmitGate}; +pub use processing::{DenoiseState, TransmitGate, VoiceActivity}; #[derive(Debug, Clone, Copy)] pub struct AudioSync { diff --git a/src/audio/pipeline/capture.rs b/src/audio/pipeline/capture.rs index a64615c..45b6977 100644 --- a/src/audio/pipeline/capture.rs +++ b/src/audio/pipeline/capture.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use cpal::traits::{DeviceTrait, StreamTrait}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; @@ -7,9 +7,13 @@ use tracing::{info, warn}; use crate::audio::{ EncodedFrame, config::{self, AudioConfig, OPUS_RATE}, - processing::{DenoiseState, TransmitGate}, + diagnostics::{AudioStage, Diagnostics}, + processing::{DenoiseState, TransmitGate, VoiceActivity}, }; +/// Seconds of exact digital silence after which the microphone is reported as dead. +const SILENCE_WARNING_SECS: u32 = 4; + const MAX_OPUS: usize = 4000; #[allow(clippy::too_many_arguments)] @@ -24,6 +28,7 @@ pub fn run( local_speaking: Arc, stopped: Arc, microphone_tap: super::media::MicrophoneTap, + diagnostics: Arc, ) -> Result<()> { let mut encoder = opus::Encoder::new(OPUS_RATE, opus::Channels::Mono, opus::Application::Voip)?; let mut last_bitrate_bps: u32 = 0; @@ -41,24 +46,38 @@ pub fn run( let mut voice_seq: u32 = 0; let mut timestamp: u32 = 0; - let stream = device.build_input_stream( - cfg.stream, - move |data: &[f32], _| { - let mut b = buf2.lock().unwrap(); - if in_channels <= 1 { - b.extend_from_slice(data); - } else { - for frame in data.chunks(in_channels) { - let sum: f32 = frame.iter().sum(); - b.push(sum / in_channels as f32); + let error_diagnostics = diagnostics.clone(); + let stream = device + .build_input_stream( + cfg.stream, + move |data: &[f32], _| { + let mut b = buf2.lock().unwrap(); + if in_channels <= 1 { + b.extend_from_slice(data); + } else { + for frame in data.chunks(in_channels) { + let sum: f32 = frame.iter().sum(); + b.push(sum / in_channels as f32); + } } - } - }, - |e| tracing::error!("input: {e}"), - None, - )?; - stream.play()?; + }, + move |e| { + error_diagnostics.report( + AudioStage::Capture, + false, + format!("microphone stream: {e}"), + ) + }, + None, + ) + .context("cannot open the microphone stream")?; + stream + .play() + .context("cannot start the microphone stream")?; info!("capture started ({} Hz)", cfg.sample_rate); + let mut vad = VoiceActivity::default(); + let mut silent_frames: u32 = 0; + let mut silence_reported = false; while !stopped.load(Ordering::Acquire) { std::thread::sleep(std::time::Duration::from_millis(5)); @@ -80,12 +99,29 @@ pub fn run( padded }; + let peak = raw_peak(&pcm); + diagnostics.count_captured(peak); + if peak == 0.0 { + silent_frames = silent_frames.saturating_add(1); + if !silence_reported && silent_frames * frame_ms >= SILENCE_WARNING_SECS * 1000 { + silence_reported = true; + diagnostics.report( + AudioStage::Capture, + false, + "the microphone delivers only silence; check the input device and its mute switch", + ); + } + } else { + silent_frames = 0; + silence_reported = false; + } + let gate_snapshot = gate.lock().unwrap().clone(); TransmitGate::apply_input_volume(&mut pcm, gate_snapshot.input_volume); if gate_snapshot.noise_suppress { denoise.process(&mut pcm); } - let speaking = TransmitGate::should_transmit(&gate_snapshot, &pcm); + let speaking = gate_snapshot.decide(&pcm, &mut vad, frame_ms); local_speaking.store(speaking, Ordering::Relaxed); if !speaking { b = buf.lock().unwrap(); @@ -110,6 +146,7 @@ pub fn run( { return Ok(()); } + diagnostics.count_transmitted(); voice_seq = voice_seq.wrapping_add(1); timestamp = timestamp.wrapping_add(opus_frame as u32); } @@ -121,6 +158,10 @@ pub fn run( Ok(()) } +fn raw_peak(samples: &[f32]) -> f32 { + samples.iter().fold(0.0f32, |peak, s| peak.max(s.abs())) +} + fn frame_samples_at(sample_rate: u32, frame_ms: u32) -> usize { (sample_rate as usize * frame_ms as usize) / 1000 } diff --git a/src/audio/pipeline/effects.rs b/src/audio/pipeline/effects.rs new file mode 100644 index 0000000..403db1f --- /dev/null +++ b/src/audio/pipeline/effects.rs @@ -0,0 +1,113 @@ +//! Short notification sounds mixed into the already open playback stream. +//! +//! No extra device is opened while a voice room is active, so effects follow the selected +//! output device and cannot conflict with exclusive hardware access. The audio callback only +//! pops samples with `try_lock`; it never waits for the UI thread. + +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, +}; + +use crate::audio::config::{OPUS_RATE, resample_rate}; + +/// Longest backlog of queued effect audio (two seconds) so a burst cannot grow without bound. +const MAX_SECONDS: usize = 2; + +#[derive(Clone)] +pub struct EffectQueue { + samples: Arc>>, + device_rate: u32, +} + +impl EffectQueue { + pub(super) fn new(device_rate: u32) -> Self { + Self { + samples: Arc::default(), + device_rate, + } + } + + /// Mixes a mono 48 kHz buffer (volume already applied) on top of what is still queued. + pub fn push(&self, mono_48k: &[f32]) { + let converted = if self.device_rate == OPUS_RATE { + mono_48k.to_vec() + } else { + resample_rate(mono_48k, OPUS_RATE, self.device_rate) + }; + let limit = self.device_rate as usize * MAX_SECONDS; + let Ok(mut queue) = self.samples.lock() else { + return; + }; + for (index, sample) in converted.into_iter().take(limit).enumerate() { + if let Some(existing) = queue.get_mut(index) { + *existing += sample; + } else { + queue.push_back(sample); + } + } + } + + /// Adds queued effect samples to an interleaved output buffer. + pub(super) fn mix_into(&self, out: &mut [f32], channels: usize) { + let Ok(mut queue) = self.samples.try_lock() else { + return; + }; + if queue.is_empty() || channels == 0 { + return; + } + for frame in out.chunks_mut(channels) { + let Some(sample) = queue.pop_front() else { + break; + }; + for value in frame { + *value = (*value + sample).clamp(-1.0, 1.0); + } + } + } + + pub(super) fn clear(&self) { + if let Ok(mut queue) = self.samples.lock() { + queue.clear(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn effects_play_once_on_every_channel_and_then_stop() { + let queue = EffectQueue::new(48_000); + queue.push(&[0.5, 0.25]); + let mut out = [0.0f32; 8]; + queue.mix_into(&mut out, 2); + assert_eq!(out, [0.5, 0.5, 0.25, 0.25, 0.0, 0.0, 0.0, 0.0]); + let mut again = [0.0f32; 4]; + queue.mix_into(&mut again, 2); + assert_eq!(again, [0.0; 4]); + } + + #[test] + fn overlapping_effects_add_and_the_output_is_clamped() { + let queue = EffectQueue::new(48_000); + queue.push(&[0.4, 0.4]); + queue.push(&[0.4, 0.4, 0.4]); + let mut out = [0.9f32; 3]; + queue.mix_into(&mut out, 1); + assert_eq!(out, [1.0, 1.0, 1.0]); + } + + #[test] + fn queue_is_bounded_and_resampled_to_the_device_rate() { + let queue = EffectQueue::new(24_000); + queue.push(&vec![0.1; 48_000]); + let length = queue.samples.lock().unwrap().len(); + assert_eq!(length, 24_000); + queue.clear(); + assert!(queue.samples.lock().unwrap().is_empty()); + queue.push(&vec![0.1; 48_000 * 10]); + assert_eq!(queue.samples.lock().unwrap().len(), 24_000 * MAX_SECONDS); + } +} diff --git a/src/audio/pipeline/mod.rs b/src/audio/pipeline/mod.rs index 1e9e0ea..1a7f4f7 100644 --- a/src/audio/pipeline/mod.rs +++ b/src/audio/pipeline/mod.rs @@ -1,4 +1,5 @@ mod capture; +mod effects; mod media; pub use media::MediaMixer; mod playback; @@ -12,6 +13,7 @@ use tracing::info; use crate::audio::{ AudioSync, EncodedFrame, IncomingVoice, PipelineConfig, config, devices, + diagnostics::{AudioIssue, AudioStage, AudioStats, Diagnostics}, processing::TransmitGate, }; @@ -22,6 +24,8 @@ pub struct AudioPipeline { encoded_rx: Receiver, microphone_tap: media::MicrophoneTap, media_mixer: MediaMixer, + effects: effects::EffectQueue, + diagnostics: Arc, voice_tx: Sender, _encoded_tx: Sender, gate: Arc>, @@ -40,6 +44,7 @@ impl Drop for AudioPipeline { self.stopped.store(true, Ordering::Release); self.microphone_tap.enable(false); self.media_mixer.clear(); + self.effects.clear(); } } @@ -54,6 +59,27 @@ impl AudioPipeline { self.media_mixer.clone() } + /// Plays a short notification (mono, 48 kHz, volume already applied) through the open + /// playback stream without touching the voice path. + pub fn play_effect(&self, mono_48k: &[f32]) { + self.effects.push(mono_48k); + } + + /// Problems raised by the capture and playback threads since the last call. + pub fn take_issues(&self) -> Vec { + self.diagnostics.take_issues() + } + + /// True when neither the microphone nor the output could be used. + pub fn is_dead(&self) -> bool { + self.diagnostics.all_failed() + } + + /// Signal counters for support logs. + pub fn stats(&self) -> AudioStats { + self.diagnostics.snapshot() + } + pub fn poll_encoded(&self) -> Option { self.encoded_rx.try_recv().ok() } @@ -124,6 +150,11 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { let tap_capture = microphone_tap.clone(); let media_mixer = MediaMixer::new(out_cfg.sample_rate); let mixer_playback = media_mixer.clone(); + let effects = effects::EffectQueue::new(out_cfg.sample_rate); + let effects_playback = effects.clone(); + let diagnostics = Arc::new(Diagnostics::default()); + let diagnostics_capture = diagnostics.clone(); + let diagnostics_playback = diagnostics.clone(); let (encoded_tx, encoded_rx) = mpsc::channel::(); let (voice_tx, voice_rx) = mpsc::channel::(); @@ -162,8 +193,9 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { local_speaking_cap, capture_stopped, tap_capture, + diagnostics_capture.clone(), ) { - tracing::error!("capture: {e}"); + diagnostics_capture.report(AudioStage::Capture, true, format!("{e:#}")); } }); std::thread::spawn(move || { @@ -175,8 +207,10 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { jitter_play, last_remote_voice_play, mixer_playback, + effects_playback, + diagnostics_playback.clone(), ) { - tracing::error!("playback: {e}"); + diagnostics_playback.report(AudioStage::Playback, true, format!("{e:#}")); } }); @@ -185,6 +219,8 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { encoded_rx, microphone_tap, media_mixer, + effects, + diagnostics, voice_tx, _encoded_tx: encoded_tx, gate, diff --git a/src/audio/pipeline/playback.rs b/src/audio/pipeline/playback.rs index 2d9291a..817c08d 100644 --- a/src/audio/pipeline/playback.rs +++ b/src/audio/pipeline/playback.rs @@ -1,5 +1,5 @@ use crate::jitter::{BufferedPacket, JitterBuffer}; -use anyhow::Result; +use anyhow::{Context, Result}; use cpal::traits::{DeviceTrait, StreamTrait}; use std::sync::{ Arc, Mutex, @@ -11,6 +11,7 @@ use tracing::{info, warn}; use crate::audio::IncomingVoice; use crate::audio::config::{self, AudioConfig, OPUS_RATE}; +use crate::audio::diagnostics::{AudioStage, Diagnostics}; #[derive(Debug, Clone)] pub struct JitterConfig { @@ -27,6 +28,8 @@ pub fn run( jitter_cfg: Arc>, last_remote_voice_ms: Arc, media_mixer: super::media::MediaMixer, + effects: super::effects::EffectQueue, + diagnostics: Arc, ) -> Result<()> { let mut decoder = opus::Decoder::new(OPUS_RATE, opus::Channels::Mono)?; let pb: Arc>> = Arc::new(Mutex::new(Vec::new())); @@ -37,51 +40,60 @@ pub fn run( let opus_frame = config::frame_samples(OPUS_RATE); let media_queues = media_mixer.queues.clone(); - let stream = device.build_output_stream( - cfg.stream, - move |out: &mut [f32], _| { - let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0); - let mut buf = pb2.lock().unwrap(); - if out_channels == 1 { - let n = buf.len().min(out.len()); - for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) { - *o = s * gain; + let error_diagnostics = diagnostics.clone(); + let effects_playback = effects.clone(); + let stream = device + .build_output_stream( + cfg.stream, + move |out: &mut [f32], _| { + let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0); + let mut buf = pb2.lock().unwrap(); + if out_channels == 1 { + let n = buf.len().min(out.len()); + for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) { + *o = s * gain; + } + buf.drain(..n); + out[n..].fill(0.0); + } else { + let frames = out.len() / out_channels; + for i in 0..frames { + let sample = if i < buf.len() { buf[i] * gain } else { 0.0 }; + for ch in 0..out_channels { + out[i * out_channels + ch] = sample; + } + } + let drain_n = frames.min(buf.len()); + buf.drain(..drain_n); } - buf.drain(..n); - out[n..].fill(0.0); - } else { - let frames = out.len() / out_channels; - for i in 0..frames { - let sample = if i < buf.len() { buf[i] * gain } else { 0.0 }; - for ch in 0..out_channels { - out[i * out_channels + ch] = sample; + effects_playback.mix_into(out, out_channels); + if let Ok(mut queues) = media_queues.lock() { + for frame in out.chunks_mut(out_channels) { + let mut mixed = [0.0f32; 2]; + for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) { + mixed[0] += samples[0]; + mixed[1] += samples[1]; + } + for (channel, sample) in frame.iter_mut().enumerate() { + let value = if out_channels == 1 || channel > 1 { + (mixed[0] + mixed[1]) * 0.5 + } else { + mixed[channel] + }; + *sample = (*sample + value * gain).clamp(-1.0, 1.0); + } } } - let drain_n = frames.min(buf.len()); - buf.drain(..drain_n); - } - if let Ok(mut queues) = media_queues.lock() { - for frame in out.chunks_mut(out_channels) { - let mut mixed = [0.0f32; 2]; - for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) { - mixed[0] += samples[0]; - mixed[1] += samples[1]; - } - for (channel, sample) in frame.iter_mut().enumerate() { - let value = if out_channels == 1 || channel > 1 { - (mixed[0] + mixed[1]) * 0.5 - } else { - mixed[channel] - }; - *sample = (*sample + value * gain).clamp(-1.0, 1.0); - } - } - } - }, - |e| tracing::error!("output: {e}"), - None, - )?; - stream.play()?; + }, + move |e| { + error_diagnostics.report(AudioStage::Playback, false, format!("output stream: {e}")) + }, + None, + ) + .context("cannot open the audio output stream")?; + stream + .play() + .context("cannot start the audio output stream")?; info!("playback started ({} Hz)", cfg.sample_rate); let clock = Instant::now(); @@ -121,6 +133,7 @@ pub fn run( opus_data: pkt.opus_data, arrived_at: clock.elapsed().as_millis() as u64, }); + diagnostics.count_received(); } Err(TryRecvError::Empty) => break, Err(TryRecvError::Disconnected) => { @@ -138,6 +151,7 @@ pub fn run( needs_resample, cfg.sample_rate, &pb, + &diagnostics, ); } std::thread::sleep(Duration::from_millis(2)); @@ -152,6 +166,7 @@ fn decode_into_playback( needs_resample: bool, device_rate: u32, pb: &Arc>>, + diagnostics: &Diagnostics, ) { let mut pcm = vec![0f32; opus_frame]; match decoder.decode_float(data, &mut pcm, false) { @@ -163,6 +178,7 @@ fn decode_into_playback( pcm }; pb.lock().unwrap().extend_from_slice(&samples); + diagnostics.count_decoded(); } Err(e) => warn!("decode: {e}"), } diff --git a/src/audio/processing/mod.rs b/src/audio/processing/mod.rs index 4375601..9e58369 100644 --- a/src/audio/processing/mod.rs +++ b/src/audio/processing/mod.rs @@ -15,8 +15,8 @@ impl Default for TransmitGate { fn default() -> Self { Self { mic_enabled: true, - vad_mode: 0, - vad_threshold: 40.0, + vad_mode: 1, + vad_threshold: 15.0, ptt_held: false, input_volume: 80.0, noise_suppress: false, @@ -62,6 +62,59 @@ fn vad_rms_threshold(ui_threshold: f32) -> f32 { 0.002 + (ui_threshold.clamp(0.0, 100.0) / 100.0) * 0.118 } +/// Lowest RMS the automatic threshold may settle on (about -48 dBFS). +const VAD_MIN_THRESHOLD: f32 = 0.004; +/// How long transmission continues after the signal drops below the threshold. +const VAD_HANGOVER_MS: u32 = 300; + +/// Voice activity detector with an automatically tracked noise floor. +/// +/// The user threshold is only a ceiling: the detector opens a little above the measured +/// background noise, so quiet microphones work without manual tuning and noisy rooms do not +/// keep the channel open. A short hangover avoids clipping word endings. +#[derive(Debug, Clone)] +pub struct VoiceActivity { + noise_floor: f32, + hangover_ms: u32, +} + +impl Default for VoiceActivity { + fn default() -> Self { + Self { + noise_floor: 0.004, + hangover_ms: 0, + } + } +} + +impl VoiceActivity { + /// Current opening threshold for the given ceiling. + pub fn threshold(&self, ceiling: f32) -> f32 { + (self.noise_floor * 3.0 + 0.002) + .max(VAD_MIN_THRESHOLD) + .min(ceiling.max(VAD_MIN_THRESHOLD)) + } + + /// Feeds one frame (`frame_ms` long) and tells whether it should be transmitted. + pub fn update(&mut self, rms: f32, ceiling: f32, frame_ms: u32) -> bool { + let threshold = self.threshold(ceiling); + if rms >= threshold { + self.hangover_ms = VAD_HANGOVER_MS; + // Sustained sound the detector cannot tell from speech slowly raises the floor, so a + // fan that starts mid-call stops holding the channel open. + self.noise_floor += (rms.min(ceiling / 3.0) - self.noise_floor).max(0.0) * 0.0005; + return true; + } + let rate = if rms < self.noise_floor { 0.2 } else { 0.02 }; + self.noise_floor += (rms - self.noise_floor) * rate; + if self.hangover_ms > 0 { + self.hangover_ms = self.hangover_ms.saturating_sub(frame_ms); + return true; + } + false + } +} + impl TransmitGate { pub fn apply_input_volume(pcm: &mut [f32], volume_pct: f32) { let gain = (volume_pct / 100.0).clamp(0.0, 2.0); @@ -73,6 +126,21 @@ impl TransmitGate { } } + /// Like [`should_transmit`](Self::should_transmit) but voice activation uses the adaptive + /// detector in `vad` instead of a fixed threshold. + pub fn decide(&self, pcm: &[f32], vad: &mut VoiceActivity, frame_ms: u32) -> bool { + if self.mic_enabled && self.vad_mode == 1 { + // The detector must see every frame to track the noise floor, even when muted + // frames are skipped above. + return vad.update( + frame_rms(pcm), + vad_rms_threshold(self.vad_threshold), + frame_ms, + ); + } + self.should_transmit(pcm) + } + pub fn should_transmit(&self, pcm: &[f32]) -> bool { if !self.mic_enabled { return false; @@ -129,6 +197,72 @@ mod tests { assert!(gate.should_transmit(&loud_frame())); } + #[test] + fn detector_adapts_to_a_quiet_microphone() { + let mut vad = VoiceActivity::default(); + // Room tone at 0.0005 RMS for one second lets the floor settle. + for _ in 0..50 { + vad.update(0.0005, 0.02, 20); + } + assert!(vad.threshold(0.02) <= VAD_MIN_THRESHOLD + 1e-6); + // Quiet speech at 0.008 RMS would never pass a fixed 0.05 threshold. + assert!(vad.update(0.008, 0.02, 20)); + } + + #[test] + fn detector_never_exceeds_the_user_ceiling() { + let mut vad = VoiceActivity::default(); + for _ in 0..2000 { + vad.update(0.5, 0.02, 20); + } + assert!(vad.threshold(0.02) <= 0.02 + 1e-6); + } + + #[test] + fn detector_keeps_transmitting_through_the_hangover_then_stops() { + let mut vad = VoiceActivity::default(); + for _ in 0..20 { + vad.update(0.0003, 0.02, 20); + } + assert!(vad.update(0.05, 0.02, 20)); + let mut open_frames = 0; + while vad.update(0.0003, 0.02, 20) { + open_frames += 1; + assert!(open_frames < 100); + } + assert_eq!(open_frames, (VAD_HANGOVER_MS / 20) as usize); + } + + #[test] + fn default_gate_uses_voice_activation_with_a_sane_ceiling() { + let gate = TransmitGate::default(); + assert_eq!(gate.vad_mode, 1); + // The ceiling must be low enough for ordinary speech levels. + assert!(vad_rms_threshold(gate.vad_threshold) < 0.03); + } + + #[test] + fn decide_gates_voice_activation_by_mic_state_and_mode() { + let mut vad = VoiceActivity::default(); + let loud = loud_frame(); + let gate = TransmitGate { + vad_mode: 1, + ..Default::default() + }; + assert!(gate.decide(&loud, &mut vad, 20)); + let muted = TransmitGate { + mic_enabled: false, + vad_mode: 1, + ..Default::default() + }; + assert!(!muted.decide(&loud, &mut vad, 20)); + let ptt = TransmitGate { + vad_mode: 0, + ..Default::default() + }; + assert!(!ptt.decide(&loud, &mut vad, 20)); + } + #[test] fn vad_uses_threshold() { let gate = TransmitGate { diff --git a/src/net/session/handlers/voice.rs b/src/net/session/handlers/voice.rs index 3da58ff..9be37cc 100644 --- a/src/net/session/handlers/voice.rs +++ b/src/net/session/handlers/voice.rs @@ -31,6 +31,7 @@ impl State { let events = event_tx.clone(); let receiver = tokio::spawn(async move { let mut bytes = vec![0u8; voice::MAX_DATAGRAM + 1]; + let mut rejected: u64 = 0; while let Ok((length, source)) = worker_udp.recv_from(&mut bytes).await { let event = { let Ok(mut state) = worker_state.lock() else { @@ -40,12 +41,29 @@ impl State { continue; }; if source != state.endpoint { + // A relay answering from another address looks exactly like silence. + rejected += 1; + if rejected == 1 || rejected % 500 == 0 { + tracing::warn!( + %source, + expected = %state.endpoint, + rejected, + "voice datagram from an unexpected address ignored" + ); + } continue; } - let Ok((sequence, frame)) = voice::decode(&state.credentials, &bytes[..length]) - else { - continue; - }; + let (sequence, frame) = + match voice::decode(&state.credentials, &bytes[..length]) { + Ok(decoded) => decoded, + Err(error) => { + rejected += 1; + if rejected == 1 || rejected % 500 == 0 { + tracing::warn!(%error, rejected, "voice datagram rejected"); + } + continue; + } + }; if !state.replay.accept(sequence) || frame.opus.is_empty() { continue; } diff --git a/src/ui/update.rs b/src/ui/update.rs index 8354e25..3e0f1ac 100644 --- a/src/ui/update.rs +++ b/src/ui/update.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use vnox_client::audio::{IncomingVoice, PipelineConfig}; +use vnox_client::audio::{AudioStage, IncomingVoice, PipelineConfig}; use vnox_client::net::{ChatMsg, NetCommand, NetEvent, NetHandle, PresenceInfo}; use crate::ui::state::{ChatMessage, ConnState, UiState, VoiceUiState}; @@ -304,11 +304,13 @@ pub fn process_net_events(state: &mut AppState) { // Capture/playback runs independently from incoming control packets. sync_voice_audio(state); flush_encoded_voice(state); + report_audio_issues(state); + log_audio_stats(state); if let Some(pipeline) = state.audio.as_ref() { pipeline.sync_controls(vnox_client::audio::AudioSync { mic_enabled: state.ui.mic_enabled && !state.ui.deafen, vad_mode: state.ui.vad_mode, - vad_threshold: 40.0, + vad_threshold: 15.0, input_volume: 100.0, ptt_held: state.ui.ptt_held, output_volume: if state.ui.deafen { @@ -950,3 +952,66 @@ pub fn flush_encoded_voice(state: &mut AppState) { }); } } + +/// Surfaces capture and playback failures instead of leaving a "connected" room that is mute. +fn report_audio_issues(state: &mut AppState) { + let Some(pipeline) = state.audio.as_ref() else { + return; + }; + let issues = pipeline.take_issues(); + let dead = pipeline.is_dead(); + for issue in issues { + let message = match issue.stage { + AudioStage::Capture => { + crate::ui::i18n::Message::with("Microphone error: {e}", vec![("e", issue.message)]) + } + AudioStage::Playback => crate::ui::i18n::Message::with( + "Audio output error: {e}", + vec![("e", issue.message)], + ), + }; + state.ui.status_msg = Some(message); + } + if dead { + let reason = state + .ui + .status_msg + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(); + state.audio = None; + state.voice_channel = None; + state.ui.voice = VoiceUiState::Error(reason); + } +} + +/// Logs signal counters every ten seconds while a voice room is active, so a bug report can +/// tell whether the signal stopped at the microphone, the network or the speaker. +fn log_audio_stats(state: &AppState) { + thread_local! { + static LAST: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + let due = LAST.with(|last| { + let due = last + .get() + .is_none_or(|t| t.elapsed() >= std::time::Duration::from_secs(10)); + if due { + last.set(Some(std::time::Instant::now())); + } + due + }); + if !due { + return; + } + if let Some(pipeline) = state.audio.as_ref() { + let stats = pipeline.stats(); + tracing::info!( + captured = stats.captured_frames, + sent = stats.transmitted_frames, + received = stats.received_packets, + decoded = stats.decoded_frames, + input_peak = stats.input_peak, + "voice audio stats" + ); + } +} diff --git a/translations/ru.json b/translations/ru.json index 5618015..27afbee 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -216,6 +216,7 @@ "Reconnecting in {delay_secs}s (attempt {attempt})": "Повторное подключение через {delay_secs} с (попытка {attempt})", "Server error: {message}": "Ошибка сервера: {message}", "Microphone error: {e}": "Ошибка микрофона: {e}", + "Audio output error: {e}": "Ошибка вывода звука: {e}", "Cannot save audio settings: {error}": "Не удалось сохранить настройки звука: {error}", "Passphrases do not match": "Пароли не совпадают", "Passphrase must be at least 8 characters": "Пароль должен содержать минимум 8 символов",