From 69d108b0909ee8ca8e9530929de0845d5ce00692 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Sun, 25 Oct 2026 00:18:03 +0200 Subject: [PATCH] fix(client): surface audio failures and use adaptive voice activation Capture and playback failures were only written to the log on detached threads, so a microphone or output that could not be opened still showed a connected voice room with no sound. Threads now report into a shared diagnostics sink that the UI polls: errors appear in the status bar, a room with neither working half is marked failed, and signal counters (captured, sent, received, decoded, input peak) are logged every ten seconds. A microphone that delivers only digital silence is reported, and rejected voice datagrams are logged instead of dropped silently. Voice activation now tracks the background noise floor and opens just above it, capped by a much lower default ceiling; the old fixed threshold (about -26 dBFS) rarely opened for normal speech. Stream configuration only accepts f32 formats and says so when a device offers nothing else. Playback gains a mixing queue for short notification effects. --- src/audio/config.rs | 101 +++++++++++---- src/audio/diagnostics.rs | 201 ++++++++++++++++++++++++++++++ src/audio/mod.rs | 4 +- src/audio/pipeline/capture.rs | 79 +++++++++--- src/audio/pipeline/effects.rs | 113 +++++++++++++++++ src/audio/pipeline/mod.rs | 40 +++++- src/audio/pipeline/playback.rs | 102 ++++++++------- src/audio/processing/mod.rs | 138 +++++++++++++++++++- src/net/session/handlers/voice.rs | 26 +++- src/ui/update.rs | 69 +++++++++- translations/ru.json | 1 + 11 files changed, 777 insertions(+), 97 deletions(-) create mode 100644 src/audio/diagnostics.rs create mode 100644 src/audio/pipeline/effects.rs diff --git a/src/audio/config.rs b/src/audio/config.rs index 304dd5f..0b4524b 100644 --- a/src/audio/config.rs +++ b/src/audio/config.rs @@ -1,6 +1,6 @@ use anyhow::{Result, anyhow}; use cpal::traits::DeviceTrait; -use cpal::{SampleFormat, StreamConfig}; +use cpal::{SampleFormat, StreamConfig, SupportedStreamConfigRange}; use tracing::info; pub const OPUS_RATE: u32 = 48_000; @@ -41,24 +41,19 @@ impl AudioConfig { device.supported_output_configs()?.collect() }; - // Prefer 48 kHz mono F32 — matches Opus native rate. - for r in &ranges { - if r.channels() == 1 - && r.sample_format() == SampleFormat::F32 - && r.min_sample_rate() <= OPUS_RATE - && r.max_sample_rate() >= OPUS_RATE - { - let stream = r.with_sample_rate(OPUS_RATE).config(); - info!( - "audio {}: {} Hz mono", - if is_input { "in" } else { "out" }, - OPUS_RATE - ); - return Ok(Self { - stream, - sample_rate: OPUS_RATE, - }); - } + // Streams are built with f32 callbacks, so only f32 configurations are usable. + // Prefer 48 kHz (Opus native rate) with the fewest channels, mono first. + let direction = if is_input { "in" } else { "out" }; + if let Some(range) = best_opus_range(&ranges) { + let stream = range.with_sample_rate(OPUS_RATE).config(); + info!( + "audio {direction}: {OPUS_RATE} Hz {}ch f32", + stream.channels + ); + return Ok(Self { + stream, + sample_rate: OPUS_RATE, + }); } // Fallback: device native rate, resample to/from Opus in capture/playback. @@ -68,14 +63,20 @@ impl AudioConfig { device.default_output_config() } .map_err(|e| anyhow!("no default audio config: {e}"))?; + if default.sample_format() != SampleFormat::F32 { + return Err(anyhow!( + "the {} device offers {:?} samples but 32-bit float is required; \ + pick another device in the audio settings", + if is_input { "input" } else { "output" }, + default.sample_format() + )); + } let sample_rate = default.sample_rate(); let stream: StreamConfig = default.config(); info!( - "audio {}: {} Hz {}ch (will resample for Opus)", - if is_input { "in" } else { "out" }, - sample_rate, - stream.channels + "audio {direction}: {} Hz {}ch (will resample for Opus)", + sample_rate, stream.channels ); Ok(Self { stream, @@ -84,6 +85,18 @@ impl AudioConfig { } } +/// Picks the f32 range that covers 48 kHz with the fewest channels (mono preferred). +fn best_opus_range(ranges: &[SupportedStreamConfigRange]) -> Option<&SupportedStreamConfigRange> { + ranges + .iter() + .filter(|r| { + r.sample_format() == SampleFormat::F32 + && r.min_sample_rate() <= OPUS_RATE + && r.max_sample_rate() >= OPUS_RATE + }) + .min_by_key(|r| (r.channels() != 1, r.channels())) +} + pub fn frame_samples(sample_rate: u32) -> usize { (sample_rate as usize * FRAME_MS as usize) / 1000 } @@ -118,7 +131,47 @@ pub fn resample_rate(input: &[f32], from_hz: u32, to_hz: u32) -> Vec { #[cfg(test)] mod tests { - use super::{bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting}; + use super::{ + OPUS_RATE, SampleFormat, SupportedStreamConfigRange, best_opus_range, + bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting, + }; + + use cpal::SupportedBufferSize; + + fn range( + channels: u16, + min: u32, + max: u32, + format: SampleFormat, + ) -> SupportedStreamConfigRange { + SupportedStreamConfigRange::new(channels, min, max, SupportedBufferSize::Unknown, format) + } + + #[test] + fn config_prefers_mono_f32_at_the_opus_rate() { + let ranges = [ + range(2, 1, 384_000, SampleFormat::F32), + range(1, 1, 384_000, SampleFormat::I16), + range(1, 1, 384_000, SampleFormat::F32), + ]; + let best = best_opus_range(&ranges).expect("f32 range"); + assert_eq!( + (best.channels(), best.sample_format()), + (1, SampleFormat::F32) + ); + } + + #[test] + fn config_falls_back_to_stereo_and_rejects_non_f32_or_wrong_rate() { + let stereo = [range(2, 8_000, 96_000, SampleFormat::F32)]; + assert_eq!(best_opus_range(&stereo).map(|r| r.channels()), Some(2)); + let unusable = [ + range(1, 8_000, 44_100, SampleFormat::F32), + range(1, 1, 96_000, SampleFormat::I16), + ]; + assert!(best_opus_range(&unusable).is_none()); + assert_eq!(OPUS_RATE, 48_000); + } #[test] fn jitter_size_maps_to_ms() { diff --git a/src/audio/diagnostics.rs b/src/audio/diagnostics.rs new file mode 100644 index 0000000..aeb0f96 --- /dev/null +++ b/src/audio/diagnostics.rs @@ -0,0 +1,201 @@ +//! Shared audio health: failures raised on capture and playback threads must reach the UI. +//! +//! Earlier versions only logged such failures (`tracing::error!` on a detached thread), so a +//! microphone that could not be opened still showed a connected voice room with no sound. The +//! threads now report into [`Diagnostics`]; the UI polls it and shows the message. + +use std::sync::{ + Mutex, + atomic::{AtomicU32, AtomicU64, Ordering}, +}; + +/// Which half of the audio pipeline raised an issue. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioStage { + Capture, + Playback, +} + +/// One reportable audio problem. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioIssue { + pub stage: AudioStage, + pub message: String, + /// The stage cannot continue (stream not built, thread ended). + pub fatal: bool, +} + +/// Counters for support logs: where in the chain did the signal stop? +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct AudioStats { + pub captured_frames: u64, + pub transmitted_frames: u64, + pub received_packets: u64, + pub decoded_frames: u64, + /// Loudest input sample seen since the last snapshot, 0.0 to 1.0. + pub input_peak: f32, +} + +#[derive(Default)] +struct Failures { + capture: bool, + playback: bool, +} + +/// Thread-safe sink for issues and counters, shared by the pipeline threads. +#[derive(Default)] +pub struct Diagnostics { + issues: Mutex>, + failed: Mutex, + last_message: Mutex>, + captured_frames: AtomicU64, + transmitted_frames: AtomicU64, + received_packets: AtomicU64, + decoded_frames: AtomicU64, + input_peak_bits: AtomicU32, +} + +impl Diagnostics { + /// Records an issue (and logs it). Identical consecutive reports from one stage are + /// collapsed so a failing device callback cannot flood the UI. + pub fn report(&self, stage: AudioStage, fatal: bool, message: impl Into) { + let message = message.into(); + if fatal && let Ok(mut failed) = self.failed.lock() { + match stage { + AudioStage::Capture => failed.capture = true, + AudioStage::Playback => failed.playback = true, + } + } + if let Ok(mut last) = self.last_message.lock() { + if last + .as_ref() + .is_some_and(|(s, m)| *s == stage && *m == message) + { + return; + } + *last = Some((stage, message.clone())); + } + tracing::error!(?stage, fatal, "audio: {message}"); + if let Ok(mut issues) = self.issues.lock() { + if issues.len() >= 16 { + issues.remove(0); + } + issues.push(AudioIssue { + stage, + message, + fatal, + }); + } + } + + /// Drains the issues reported since the last call. + pub fn take_issues(&self) -> Vec { + self.issues + .lock() + .map(|mut issues| std::mem::take(&mut *issues)) + .unwrap_or_default() + } + + /// True when neither capture nor playback is working any more. + pub fn all_failed(&self) -> bool { + self.failed + .lock() + .map(|failed| failed.capture && failed.playback) + .unwrap_or(false) + } + + pub fn count_captured(&self, peak: f32) { + self.captured_frames.fetch_add(1, Ordering::Relaxed); + let mut current = self.input_peak_bits.load(Ordering::Relaxed); + while peak > f32::from_bits(current) { + match self.input_peak_bits.compare_exchange_weak( + current, + peak.to_bits(), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + + pub fn count_transmitted(&self) { + self.transmitted_frames.fetch_add(1, Ordering::Relaxed); + } + + pub fn count_received(&self) { + self.received_packets.fetch_add(1, Ordering::Relaxed); + } + + pub fn count_decoded(&self) { + self.decoded_frames.fetch_add(1, Ordering::Relaxed); + } + + /// Current counters; the input peak is reset so each snapshot covers one interval. + pub fn snapshot(&self) -> AudioStats { + AudioStats { + captured_frames: self.captured_frames.load(Ordering::Relaxed), + transmitted_frames: self.transmitted_frames.load(Ordering::Relaxed), + received_packets: self.received_packets.load(Ordering::Relaxed), + decoded_frames: self.decoded_frames.load(Ordering::Relaxed), + input_peak: f32::from_bits(self.input_peak_bits.swap(0, Ordering::Relaxed)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn issues_are_delivered_once_and_duplicates_collapse() { + let diag = Diagnostics::default(); + diag.report(AudioStage::Capture, true, "no microphone"); + diag.report(AudioStage::Capture, true, "no microphone"); + diag.report(AudioStage::Playback, false, "xrun"); + let issues = diag.take_issues(); + assert_eq!(issues.len(), 2); + assert_eq!(issues[0].stage, AudioStage::Capture); + assert!(issues[0].fatal); + assert!(diag.take_issues().is_empty()); + } + + #[test] + fn all_failed_requires_both_stages_to_fail_fatally() { + let diag = Diagnostics::default(); + diag.report(AudioStage::Capture, true, "a"); + assert!(!diag.all_failed()); + diag.report(AudioStage::Playback, false, "b"); + assert!(!diag.all_failed()); + diag.report(AudioStage::Playback, true, "c"); + assert!(diag.all_failed()); + } + + #[test] + fn snapshot_reports_counters_and_resets_the_peak() { + let diag = Diagnostics::default(); + diag.count_captured(0.2); + diag.count_captured(0.5); + diag.count_captured(0.1); + diag.count_transmitted(); + diag.count_received(); + diag.count_decoded(); + let first = diag.snapshot(); + assert_eq!(first.captured_frames, 3); + assert_eq!(first.transmitted_frames, 1); + assert_eq!(first.received_packets, 1); + assert_eq!(first.decoded_frames, 1); + assert!((first.input_peak - 0.5).abs() < f32::EPSILON); + assert_eq!(diag.snapshot().input_peak, 0.0); + } + + #[test] + fn issue_queue_is_bounded() { + let diag = Diagnostics::default(); + for i in 0..40 { + diag.report(AudioStage::Playback, false, format!("e{i}")); + } + assert_eq!(diag.take_issues().len(), 16); + } +} diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 0b9b3a8..97b9b4e 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -1,12 +1,14 @@ pub mod config; pub mod devices; +pub mod diagnostics; mod pipeline; mod processing; pub use devices::{DeviceEntry, DeviceLists}; +pub use diagnostics::{AudioIssue, AudioStage, AudioStats}; pub use pipeline::JitterConfig; pub use pipeline::{AudioPipeline, MediaMixer}; -pub use processing::{DenoiseState, TransmitGate}; +pub use processing::{DenoiseState, TransmitGate, VoiceActivity}; #[derive(Debug, Clone, Copy)] pub struct AudioSync { diff --git a/src/audio/pipeline/capture.rs b/src/audio/pipeline/capture.rs index a64615c..45b6977 100644 --- a/src/audio/pipeline/capture.rs +++ b/src/audio/pipeline/capture.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use cpal::traits::{DeviceTrait, StreamTrait}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; @@ -7,9 +7,13 @@ use tracing::{info, warn}; use crate::audio::{ EncodedFrame, config::{self, AudioConfig, OPUS_RATE}, - processing::{DenoiseState, TransmitGate}, + diagnostics::{AudioStage, Diagnostics}, + processing::{DenoiseState, TransmitGate, VoiceActivity}, }; +/// Seconds of exact digital silence after which the microphone is reported as dead. +const SILENCE_WARNING_SECS: u32 = 4; + const MAX_OPUS: usize = 4000; #[allow(clippy::too_many_arguments)] @@ -24,6 +28,7 @@ pub fn run( local_speaking: Arc, stopped: Arc, microphone_tap: super::media::MicrophoneTap, + diagnostics: Arc, ) -> Result<()> { let mut encoder = opus::Encoder::new(OPUS_RATE, opus::Channels::Mono, opus::Application::Voip)?; let mut last_bitrate_bps: u32 = 0; @@ -41,24 +46,38 @@ pub fn run( let mut voice_seq: u32 = 0; let mut timestamp: u32 = 0; - let stream = device.build_input_stream( - cfg.stream, - move |data: &[f32], _| { - let mut b = buf2.lock().unwrap(); - if in_channels <= 1 { - b.extend_from_slice(data); - } else { - for frame in data.chunks(in_channels) { - let sum: f32 = frame.iter().sum(); - b.push(sum / in_channels as f32); + let error_diagnostics = diagnostics.clone(); + let stream = device + .build_input_stream( + cfg.stream, + move |data: &[f32], _| { + let mut b = buf2.lock().unwrap(); + if in_channels <= 1 { + b.extend_from_slice(data); + } else { + for frame in data.chunks(in_channels) { + let sum: f32 = frame.iter().sum(); + b.push(sum / in_channels as f32); + } } - } - }, - |e| tracing::error!("input: {e}"), - None, - )?; - stream.play()?; + }, + move |e| { + error_diagnostics.report( + AudioStage::Capture, + false, + format!("microphone stream: {e}"), + ) + }, + None, + ) + .context("cannot open the microphone stream")?; + stream + .play() + .context("cannot start the microphone stream")?; info!("capture started ({} Hz)", cfg.sample_rate); + let mut vad = VoiceActivity::default(); + let mut silent_frames: u32 = 0; + let mut silence_reported = false; while !stopped.load(Ordering::Acquire) { std::thread::sleep(std::time::Duration::from_millis(5)); @@ -80,12 +99,29 @@ pub fn run( padded }; + let peak = raw_peak(&pcm); + diagnostics.count_captured(peak); + if peak == 0.0 { + silent_frames = silent_frames.saturating_add(1); + if !silence_reported && silent_frames * frame_ms >= SILENCE_WARNING_SECS * 1000 { + silence_reported = true; + diagnostics.report( + AudioStage::Capture, + false, + "the microphone delivers only silence; check the input device and its mute switch", + ); + } + } else { + silent_frames = 0; + silence_reported = false; + } + let gate_snapshot = gate.lock().unwrap().clone(); TransmitGate::apply_input_volume(&mut pcm, gate_snapshot.input_volume); if gate_snapshot.noise_suppress { denoise.process(&mut pcm); } - let speaking = TransmitGate::should_transmit(&gate_snapshot, &pcm); + let speaking = gate_snapshot.decide(&pcm, &mut vad, frame_ms); local_speaking.store(speaking, Ordering::Relaxed); if !speaking { b = buf.lock().unwrap(); @@ -110,6 +146,7 @@ pub fn run( { return Ok(()); } + diagnostics.count_transmitted(); voice_seq = voice_seq.wrapping_add(1); timestamp = timestamp.wrapping_add(opus_frame as u32); } @@ -121,6 +158,10 @@ pub fn run( Ok(()) } +fn raw_peak(samples: &[f32]) -> f32 { + samples.iter().fold(0.0f32, |peak, s| peak.max(s.abs())) +} + fn frame_samples_at(sample_rate: u32, frame_ms: u32) -> usize { (sample_rate as usize * frame_ms as usize) / 1000 } diff --git a/src/audio/pipeline/effects.rs b/src/audio/pipeline/effects.rs new file mode 100644 index 0000000..403db1f --- /dev/null +++ b/src/audio/pipeline/effects.rs @@ -0,0 +1,113 @@ +//! Short notification sounds mixed into the already open playback stream. +//! +//! No extra device is opened while a voice room is active, so effects follow the selected +//! output device and cannot conflict with exclusive hardware access. The audio callback only +//! pops samples with `try_lock`; it never waits for the UI thread. + +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, +}; + +use crate::audio::config::{OPUS_RATE, resample_rate}; + +/// Longest backlog of queued effect audio (two seconds) so a burst cannot grow without bound. +const MAX_SECONDS: usize = 2; + +#[derive(Clone)] +pub struct EffectQueue { + samples: Arc>>, + device_rate: u32, +} + +impl EffectQueue { + pub(super) fn new(device_rate: u32) -> Self { + Self { + samples: Arc::default(), + device_rate, + } + } + + /// Mixes a mono 48 kHz buffer (volume already applied) on top of what is still queued. + pub fn push(&self, mono_48k: &[f32]) { + let converted = if self.device_rate == OPUS_RATE { + mono_48k.to_vec() + } else { + resample_rate(mono_48k, OPUS_RATE, self.device_rate) + }; + let limit = self.device_rate as usize * MAX_SECONDS; + let Ok(mut queue) = self.samples.lock() else { + return; + }; + for (index, sample) in converted.into_iter().take(limit).enumerate() { + if let Some(existing) = queue.get_mut(index) { + *existing += sample; + } else { + queue.push_back(sample); + } + } + } + + /// Adds queued effect samples to an interleaved output buffer. + pub(super) fn mix_into(&self, out: &mut [f32], channels: usize) { + let Ok(mut queue) = self.samples.try_lock() else { + return; + }; + if queue.is_empty() || channels == 0 { + return; + } + for frame in out.chunks_mut(channels) { + let Some(sample) = queue.pop_front() else { + break; + }; + for value in frame { + *value = (*value + sample).clamp(-1.0, 1.0); + } + } + } + + pub(super) fn clear(&self) { + if let Ok(mut queue) = self.samples.lock() { + queue.clear(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn effects_play_once_on_every_channel_and_then_stop() { + let queue = EffectQueue::new(48_000); + queue.push(&[0.5, 0.25]); + let mut out = [0.0f32; 8]; + queue.mix_into(&mut out, 2); + assert_eq!(out, [0.5, 0.5, 0.25, 0.25, 0.0, 0.0, 0.0, 0.0]); + let mut again = [0.0f32; 4]; + queue.mix_into(&mut again, 2); + assert_eq!(again, [0.0; 4]); + } + + #[test] + fn overlapping_effects_add_and_the_output_is_clamped() { + let queue = EffectQueue::new(48_000); + queue.push(&[0.4, 0.4]); + queue.push(&[0.4, 0.4, 0.4]); + let mut out = [0.9f32; 3]; + queue.mix_into(&mut out, 1); + assert_eq!(out, [1.0, 1.0, 1.0]); + } + + #[test] + fn queue_is_bounded_and_resampled_to_the_device_rate() { + let queue = EffectQueue::new(24_000); + queue.push(&vec![0.1; 48_000]); + let length = queue.samples.lock().unwrap().len(); + assert_eq!(length, 24_000); + queue.clear(); + assert!(queue.samples.lock().unwrap().is_empty()); + queue.push(&vec![0.1; 48_000 * 10]); + assert_eq!(queue.samples.lock().unwrap().len(), 24_000 * MAX_SECONDS); + } +} diff --git a/src/audio/pipeline/mod.rs b/src/audio/pipeline/mod.rs index 1e9e0ea..1a7f4f7 100644 --- a/src/audio/pipeline/mod.rs +++ b/src/audio/pipeline/mod.rs @@ -1,4 +1,5 @@ mod capture; +mod effects; mod media; pub use media::MediaMixer; mod playback; @@ -12,6 +13,7 @@ use tracing::info; use crate::audio::{ AudioSync, EncodedFrame, IncomingVoice, PipelineConfig, config, devices, + diagnostics::{AudioIssue, AudioStage, AudioStats, Diagnostics}, processing::TransmitGate, }; @@ -22,6 +24,8 @@ pub struct AudioPipeline { encoded_rx: Receiver, microphone_tap: media::MicrophoneTap, media_mixer: MediaMixer, + effects: effects::EffectQueue, + diagnostics: Arc, voice_tx: Sender, _encoded_tx: Sender, gate: Arc>, @@ -40,6 +44,7 @@ impl Drop for AudioPipeline { self.stopped.store(true, Ordering::Release); self.microphone_tap.enable(false); self.media_mixer.clear(); + self.effects.clear(); } } @@ -54,6 +59,27 @@ impl AudioPipeline { self.media_mixer.clone() } + /// Plays a short notification (mono, 48 kHz, volume already applied) through the open + /// playback stream without touching the voice path. + pub fn play_effect(&self, mono_48k: &[f32]) { + self.effects.push(mono_48k); + } + + /// Problems raised by the capture and playback threads since the last call. + pub fn take_issues(&self) -> Vec { + self.diagnostics.take_issues() + } + + /// True when neither the microphone nor the output could be used. + pub fn is_dead(&self) -> bool { + self.diagnostics.all_failed() + } + + /// Signal counters for support logs. + pub fn stats(&self) -> AudioStats { + self.diagnostics.snapshot() + } + pub fn poll_encoded(&self) -> Option { self.encoded_rx.try_recv().ok() } @@ -124,6 +150,11 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { let tap_capture = microphone_tap.clone(); let media_mixer = MediaMixer::new(out_cfg.sample_rate); let mixer_playback = media_mixer.clone(); + let effects = effects::EffectQueue::new(out_cfg.sample_rate); + let effects_playback = effects.clone(); + let diagnostics = Arc::new(Diagnostics::default()); + let diagnostics_capture = diagnostics.clone(); + let diagnostics_playback = diagnostics.clone(); let (encoded_tx, encoded_rx) = mpsc::channel::(); let (voice_tx, voice_rx) = mpsc::channel::(); @@ -162,8 +193,9 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { local_speaking_cap, capture_stopped, tap_capture, + diagnostics_capture.clone(), ) { - tracing::error!("capture: {e}"); + diagnostics_capture.report(AudioStage::Capture, true, format!("{e:#}")); } }); std::thread::spawn(move || { @@ -175,8 +207,10 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { jitter_play, last_remote_voice_play, mixer_playback, + effects_playback, + diagnostics_playback.clone(), ) { - tracing::error!("playback: {e}"); + diagnostics_playback.report(AudioStage::Playback, true, format!("{e:#}")); } }); @@ -185,6 +219,8 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { encoded_rx, microphone_tap, media_mixer, + effects, + diagnostics, voice_tx, _encoded_tx: encoded_tx, gate, diff --git a/src/audio/pipeline/playback.rs b/src/audio/pipeline/playback.rs index 2d9291a..817c08d 100644 --- a/src/audio/pipeline/playback.rs +++ b/src/audio/pipeline/playback.rs @@ -1,5 +1,5 @@ use crate::jitter::{BufferedPacket, JitterBuffer}; -use anyhow::Result; +use anyhow::{Context, Result}; use cpal::traits::{DeviceTrait, StreamTrait}; use std::sync::{ Arc, Mutex, @@ -11,6 +11,7 @@ use tracing::{info, warn}; use crate::audio::IncomingVoice; use crate::audio::config::{self, AudioConfig, OPUS_RATE}; +use crate::audio::diagnostics::{AudioStage, Diagnostics}; #[derive(Debug, Clone)] pub struct JitterConfig { @@ -27,6 +28,8 @@ pub fn run( jitter_cfg: Arc>, last_remote_voice_ms: Arc, media_mixer: super::media::MediaMixer, + effects: super::effects::EffectQueue, + diagnostics: Arc, ) -> Result<()> { let mut decoder = opus::Decoder::new(OPUS_RATE, opus::Channels::Mono)?; let pb: Arc>> = Arc::new(Mutex::new(Vec::new())); @@ -37,51 +40,60 @@ pub fn run( let opus_frame = config::frame_samples(OPUS_RATE); let media_queues = media_mixer.queues.clone(); - let stream = device.build_output_stream( - cfg.stream, - move |out: &mut [f32], _| { - let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0); - let mut buf = pb2.lock().unwrap(); - if out_channels == 1 { - let n = buf.len().min(out.len()); - for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) { - *o = s * gain; + let error_diagnostics = diagnostics.clone(); + let effects_playback = effects.clone(); + let stream = device + .build_output_stream( + cfg.stream, + move |out: &mut [f32], _| { + let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0); + let mut buf = pb2.lock().unwrap(); + if out_channels == 1 { + let n = buf.len().min(out.len()); + for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) { + *o = s * gain; + } + buf.drain(..n); + out[n..].fill(0.0); + } else { + let frames = out.len() / out_channels; + for i in 0..frames { + let sample = if i < buf.len() { buf[i] * gain } else { 0.0 }; + for ch in 0..out_channels { + out[i * out_channels + ch] = sample; + } + } + let drain_n = frames.min(buf.len()); + buf.drain(..drain_n); } - buf.drain(..n); - out[n..].fill(0.0); - } else { - let frames = out.len() / out_channels; - for i in 0..frames { - let sample = if i < buf.len() { buf[i] * gain } else { 0.0 }; - for ch in 0..out_channels { - out[i * out_channels + ch] = sample; + effects_playback.mix_into(out, out_channels); + if let Ok(mut queues) = media_queues.lock() { + for frame in out.chunks_mut(out_channels) { + let mut mixed = [0.0f32; 2]; + for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) { + mixed[0] += samples[0]; + mixed[1] += samples[1]; + } + for (channel, sample) in frame.iter_mut().enumerate() { + let value = if out_channels == 1 || channel > 1 { + (mixed[0] + mixed[1]) * 0.5 + } else { + mixed[channel] + }; + *sample = (*sample + value * gain).clamp(-1.0, 1.0); + } } } - let drain_n = frames.min(buf.len()); - buf.drain(..drain_n); - } - if let Ok(mut queues) = media_queues.lock() { - for frame in out.chunks_mut(out_channels) { - let mut mixed = [0.0f32; 2]; - for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) { - mixed[0] += samples[0]; - mixed[1] += samples[1]; - } - for (channel, sample) in frame.iter_mut().enumerate() { - let value = if out_channels == 1 || channel > 1 { - (mixed[0] + mixed[1]) * 0.5 - } else { - mixed[channel] - }; - *sample = (*sample + value * gain).clamp(-1.0, 1.0); - } - } - } - }, - |e| tracing::error!("output: {e}"), - None, - )?; - stream.play()?; + }, + move |e| { + error_diagnostics.report(AudioStage::Playback, false, format!("output stream: {e}")) + }, + None, + ) + .context("cannot open the audio output stream")?; + stream + .play() + .context("cannot start the audio output stream")?; info!("playback started ({} Hz)", cfg.sample_rate); let clock = Instant::now(); @@ -121,6 +133,7 @@ pub fn run( opus_data: pkt.opus_data, arrived_at: clock.elapsed().as_millis() as u64, }); + diagnostics.count_received(); } Err(TryRecvError::Empty) => break, Err(TryRecvError::Disconnected) => { @@ -138,6 +151,7 @@ pub fn run( needs_resample, cfg.sample_rate, &pb, + &diagnostics, ); } std::thread::sleep(Duration::from_millis(2)); @@ -152,6 +166,7 @@ fn decode_into_playback( needs_resample: bool, device_rate: u32, pb: &Arc>>, + diagnostics: &Diagnostics, ) { let mut pcm = vec![0f32; opus_frame]; match decoder.decode_float(data, &mut pcm, false) { @@ -163,6 +178,7 @@ fn decode_into_playback( pcm }; pb.lock().unwrap().extend_from_slice(&samples); + diagnostics.count_decoded(); } Err(e) => warn!("decode: {e}"), } diff --git a/src/audio/processing/mod.rs b/src/audio/processing/mod.rs index 4375601..9e58369 100644 --- a/src/audio/processing/mod.rs +++ b/src/audio/processing/mod.rs @@ -15,8 +15,8 @@ impl Default for TransmitGate { fn default() -> Self { Self { mic_enabled: true, - vad_mode: 0, - vad_threshold: 40.0, + vad_mode: 1, + vad_threshold: 15.0, ptt_held: false, input_volume: 80.0, noise_suppress: false, @@ -62,6 +62,59 @@ fn vad_rms_threshold(ui_threshold: f32) -> f32 { 0.002 + (ui_threshold.clamp(0.0, 100.0) / 100.0) * 0.118 } +/// Lowest RMS the automatic threshold may settle on (about -48 dBFS). +const VAD_MIN_THRESHOLD: f32 = 0.004; +/// How long transmission continues after the signal drops below the threshold. +const VAD_HANGOVER_MS: u32 = 300; + +/// Voice activity detector with an automatically tracked noise floor. +/// +/// The user threshold is only a ceiling: the detector opens a little above the measured +/// background noise, so quiet microphones work without manual tuning and noisy rooms do not +/// keep the channel open. A short hangover avoids clipping word endings. +#[derive(Debug, Clone)] +pub struct VoiceActivity { + noise_floor: f32, + hangover_ms: u32, +} + +impl Default for VoiceActivity { + fn default() -> Self { + Self { + noise_floor: 0.004, + hangover_ms: 0, + } + } +} + +impl VoiceActivity { + /// Current opening threshold for the given ceiling. + pub fn threshold(&self, ceiling: f32) -> f32 { + (self.noise_floor * 3.0 + 0.002) + .max(VAD_MIN_THRESHOLD) + .min(ceiling.max(VAD_MIN_THRESHOLD)) + } + + /// Feeds one frame (`frame_ms` long) and tells whether it should be transmitted. + pub fn update(&mut self, rms: f32, ceiling: f32, frame_ms: u32) -> bool { + let threshold = self.threshold(ceiling); + if rms >= threshold { + self.hangover_ms = VAD_HANGOVER_MS; + // Sustained sound the detector cannot tell from speech slowly raises the floor, so a + // fan that starts mid-call stops holding the channel open. + self.noise_floor += (rms.min(ceiling / 3.0) - self.noise_floor).max(0.0) * 0.0005; + return true; + } + let rate = if rms < self.noise_floor { 0.2 } else { 0.02 }; + self.noise_floor += (rms - self.noise_floor) * rate; + if self.hangover_ms > 0 { + self.hangover_ms = self.hangover_ms.saturating_sub(frame_ms); + return true; + } + false + } +} + impl TransmitGate { pub fn apply_input_volume(pcm: &mut [f32], volume_pct: f32) { let gain = (volume_pct / 100.0).clamp(0.0, 2.0); @@ -73,6 +126,21 @@ impl TransmitGate { } } + /// Like [`should_transmit`](Self::should_transmit) but voice activation uses the adaptive + /// detector in `vad` instead of a fixed threshold. + pub fn decide(&self, pcm: &[f32], vad: &mut VoiceActivity, frame_ms: u32) -> bool { + if self.mic_enabled && self.vad_mode == 1 { + // The detector must see every frame to track the noise floor, even when muted + // frames are skipped above. + return vad.update( + frame_rms(pcm), + vad_rms_threshold(self.vad_threshold), + frame_ms, + ); + } + self.should_transmit(pcm) + } + pub fn should_transmit(&self, pcm: &[f32]) -> bool { if !self.mic_enabled { return false; @@ -129,6 +197,72 @@ mod tests { assert!(gate.should_transmit(&loud_frame())); } + #[test] + fn detector_adapts_to_a_quiet_microphone() { + let mut vad = VoiceActivity::default(); + // Room tone at 0.0005 RMS for one second lets the floor settle. + for _ in 0..50 { + vad.update(0.0005, 0.02, 20); + } + assert!(vad.threshold(0.02) <= VAD_MIN_THRESHOLD + 1e-6); + // Quiet speech at 0.008 RMS would never pass a fixed 0.05 threshold. + assert!(vad.update(0.008, 0.02, 20)); + } + + #[test] + fn detector_never_exceeds_the_user_ceiling() { + let mut vad = VoiceActivity::default(); + for _ in 0..2000 { + vad.update(0.5, 0.02, 20); + } + assert!(vad.threshold(0.02) <= 0.02 + 1e-6); + } + + #[test] + fn detector_keeps_transmitting_through_the_hangover_then_stops() { + let mut vad = VoiceActivity::default(); + for _ in 0..20 { + vad.update(0.0003, 0.02, 20); + } + assert!(vad.update(0.05, 0.02, 20)); + let mut open_frames = 0; + while vad.update(0.0003, 0.02, 20) { + open_frames += 1; + assert!(open_frames < 100); + } + assert_eq!(open_frames, (VAD_HANGOVER_MS / 20) as usize); + } + + #[test] + fn default_gate_uses_voice_activation_with_a_sane_ceiling() { + let gate = TransmitGate::default(); + assert_eq!(gate.vad_mode, 1); + // The ceiling must be low enough for ordinary speech levels. + assert!(vad_rms_threshold(gate.vad_threshold) < 0.03); + } + + #[test] + fn decide_gates_voice_activation_by_mic_state_and_mode() { + let mut vad = VoiceActivity::default(); + let loud = loud_frame(); + let gate = TransmitGate { + vad_mode: 1, + ..Default::default() + }; + assert!(gate.decide(&loud, &mut vad, 20)); + let muted = TransmitGate { + mic_enabled: false, + vad_mode: 1, + ..Default::default() + }; + assert!(!muted.decide(&loud, &mut vad, 20)); + let ptt = TransmitGate { + vad_mode: 0, + ..Default::default() + }; + assert!(!ptt.decide(&loud, &mut vad, 20)); + } + #[test] fn vad_uses_threshold() { let gate = TransmitGate { diff --git a/src/net/session/handlers/voice.rs b/src/net/session/handlers/voice.rs index 3da58ff..9be37cc 100644 --- a/src/net/session/handlers/voice.rs +++ b/src/net/session/handlers/voice.rs @@ -31,6 +31,7 @@ impl State { let events = event_tx.clone(); let receiver = tokio::spawn(async move { let mut bytes = vec![0u8; voice::MAX_DATAGRAM + 1]; + let mut rejected: u64 = 0; while let Ok((length, source)) = worker_udp.recv_from(&mut bytes).await { let event = { let Ok(mut state) = worker_state.lock() else { @@ -40,12 +41,29 @@ impl State { continue; }; if source != state.endpoint { + // A relay answering from another address looks exactly like silence. + rejected += 1; + if rejected == 1 || rejected % 500 == 0 { + tracing::warn!( + %source, + expected = %state.endpoint, + rejected, + "voice datagram from an unexpected address ignored" + ); + } continue; } - let Ok((sequence, frame)) = voice::decode(&state.credentials, &bytes[..length]) - else { - continue; - }; + let (sequence, frame) = + match voice::decode(&state.credentials, &bytes[..length]) { + Ok(decoded) => decoded, + Err(error) => { + rejected += 1; + if rejected == 1 || rejected % 500 == 0 { + tracing::warn!(%error, rejected, "voice datagram rejected"); + } + continue; + } + }; if !state.replay.accept(sequence) || frame.opus.is_empty() { continue; } diff --git a/src/ui/update.rs b/src/ui/update.rs index 8354e25..3e0f1ac 100644 --- a/src/ui/update.rs +++ b/src/ui/update.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use vnox_client::audio::{IncomingVoice, PipelineConfig}; +use vnox_client::audio::{AudioStage, IncomingVoice, PipelineConfig}; use vnox_client::net::{ChatMsg, NetCommand, NetEvent, NetHandle, PresenceInfo}; use crate::ui::state::{ChatMessage, ConnState, UiState, VoiceUiState}; @@ -304,11 +304,13 @@ pub fn process_net_events(state: &mut AppState) { // Capture/playback runs independently from incoming control packets. sync_voice_audio(state); flush_encoded_voice(state); + report_audio_issues(state); + log_audio_stats(state); if let Some(pipeline) = state.audio.as_ref() { pipeline.sync_controls(vnox_client::audio::AudioSync { mic_enabled: state.ui.mic_enabled && !state.ui.deafen, vad_mode: state.ui.vad_mode, - vad_threshold: 40.0, + vad_threshold: 15.0, input_volume: 100.0, ptt_held: state.ui.ptt_held, output_volume: if state.ui.deafen { @@ -950,3 +952,66 @@ pub fn flush_encoded_voice(state: &mut AppState) { }); } } + +/// Surfaces capture and playback failures instead of leaving a "connected" room that is mute. +fn report_audio_issues(state: &mut AppState) { + let Some(pipeline) = state.audio.as_ref() else { + return; + }; + let issues = pipeline.take_issues(); + let dead = pipeline.is_dead(); + for issue in issues { + let message = match issue.stage { + AudioStage::Capture => { + crate::ui::i18n::Message::with("Microphone error: {e}", vec![("e", issue.message)]) + } + AudioStage::Playback => crate::ui::i18n::Message::with( + "Audio output error: {e}", + vec![("e", issue.message)], + ), + }; + state.ui.status_msg = Some(message); + } + if dead { + let reason = state + .ui + .status_msg + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(); + state.audio = None; + state.voice_channel = None; + state.ui.voice = VoiceUiState::Error(reason); + } +} + +/// Logs signal counters every ten seconds while a voice room is active, so a bug report can +/// tell whether the signal stopped at the microphone, the network or the speaker. +fn log_audio_stats(state: &AppState) { + thread_local! { + static LAST: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + let due = LAST.with(|last| { + let due = last + .get() + .is_none_or(|t| t.elapsed() >= std::time::Duration::from_secs(10)); + if due { + last.set(Some(std::time::Instant::now())); + } + due + }); + if !due { + return; + } + if let Some(pipeline) = state.audio.as_ref() { + let stats = pipeline.stats(); + tracing::info!( + captured = stats.captured_frames, + sent = stats.transmitted_frames, + received = stats.received_packets, + decoded = stats.decoded_frames, + input_peak = stats.input_peak, + "voice audio stats" + ); + } +} diff --git a/translations/ru.json b/translations/ru.json index 5618015..27afbee 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -216,6 +216,7 @@ "Reconnecting in {delay_secs}s (attempt {attempt})": "Повторное подключение через {delay_secs} с (попытка {attempt})", "Server error: {message}": "Ошибка сервера: {message}", "Microphone error: {e}": "Ошибка микрофона: {e}", + "Audio output error: {e}": "Ошибка вывода звука: {e}", "Cannot save audio settings: {error}": "Не удалось сохранить настройки звука: {error}", "Passphrases do not match": "Пароли не совпадают", "Passphrase must be at least 8 characters": "Пароль должен содержать минимум 8 символов",