fix(client): surface audio failures and use adaptive voice activation
Capture and playback failures were only written to the log on detached threads, so a microphone or output that could not be opened still showed a connected voice room with no sound. Threads now report into a shared diagnostics sink that the UI polls: errors appear in the status bar, a room with neither working half is marked failed, and signal counters (captured, sent, received, decoded, input peak) are logged every ten seconds. A microphone that delivers only digital silence is reported, and rejected voice datagrams are logged instead of dropped silently. Voice activation now tracks the background noise floor and opens just above it, capped by a much lower default ceiling; the old fixed threshold (about -26 dBFS) rarely opened for normal speech. Stream configuration only accepts f32 formats and says so when a device offers nothing else. Playback gains a mixing queue for short notification effects.
This commit is contained in:
parent
3b95df9e9c
commit
69d108b090
11 changed files with 777 additions and 97 deletions
|
|
@ -1,6 +1,6 @@
|
||||||
use anyhow::{Result, anyhow};
|
use anyhow::{Result, anyhow};
|
||||||
use cpal::traits::DeviceTrait;
|
use cpal::traits::DeviceTrait;
|
||||||
use cpal::{SampleFormat, StreamConfig};
|
use cpal::{SampleFormat, StreamConfig, SupportedStreamConfigRange};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
pub const OPUS_RATE: u32 = 48_000;
|
pub const OPUS_RATE: u32 = 48_000;
|
||||||
|
|
@ -41,24 +41,19 @@ impl AudioConfig {
|
||||||
device.supported_output_configs()?.collect()
|
device.supported_output_configs()?.collect()
|
||||||
};
|
};
|
||||||
|
|
||||||
// Prefer 48 kHz mono F32 — matches Opus native rate.
|
// Streams are built with f32 callbacks, so only f32 configurations are usable.
|
||||||
for r in &ranges {
|
// Prefer 48 kHz (Opus native rate) with the fewest channels, mono first.
|
||||||
if r.channels() == 1
|
let direction = if is_input { "in" } else { "out" };
|
||||||
&& r.sample_format() == SampleFormat::F32
|
if let Some(range) = best_opus_range(&ranges) {
|
||||||
&& r.min_sample_rate() <= OPUS_RATE
|
let stream = range.with_sample_rate(OPUS_RATE).config();
|
||||||
&& r.max_sample_rate() >= OPUS_RATE
|
info!(
|
||||||
{
|
"audio {direction}: {OPUS_RATE} Hz {}ch f32",
|
||||||
let stream = r.with_sample_rate(OPUS_RATE).config();
|
stream.channels
|
||||||
info!(
|
);
|
||||||
"audio {}: {} Hz mono",
|
return Ok(Self {
|
||||||
if is_input { "in" } else { "out" },
|
stream,
|
||||||
OPUS_RATE
|
sample_rate: OPUS_RATE,
|
||||||
);
|
});
|
||||||
return Ok(Self {
|
|
||||||
stream,
|
|
||||||
sample_rate: OPUS_RATE,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: device native rate, resample to/from Opus in capture/playback.
|
// Fallback: device native rate, resample to/from Opus in capture/playback.
|
||||||
|
|
@ -68,14 +63,20 @@ impl AudioConfig {
|
||||||
device.default_output_config()
|
device.default_output_config()
|
||||||
}
|
}
|
||||||
.map_err(|e| anyhow!("no default audio config: {e}"))?;
|
.map_err(|e| anyhow!("no default audio config: {e}"))?;
|
||||||
|
if default.sample_format() != SampleFormat::F32 {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"the {} device offers {:?} samples but 32-bit float is required; \
|
||||||
|
pick another device in the audio settings",
|
||||||
|
if is_input { "input" } else { "output" },
|
||||||
|
default.sample_format()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let sample_rate = default.sample_rate();
|
let sample_rate = default.sample_rate();
|
||||||
let stream: StreamConfig = default.config();
|
let stream: StreamConfig = default.config();
|
||||||
info!(
|
info!(
|
||||||
"audio {}: {} Hz {}ch (will resample for Opus)",
|
"audio {direction}: {} Hz {}ch (will resample for Opus)",
|
||||||
if is_input { "in" } else { "out" },
|
sample_rate, stream.channels
|
||||||
sample_rate,
|
|
||||||
stream.channels
|
|
||||||
);
|
);
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
stream,
|
stream,
|
||||||
|
|
@ -84,6 +85,18 @@ impl AudioConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Picks the f32 range that covers 48 kHz with the fewest channels (mono preferred).
|
||||||
|
fn best_opus_range(ranges: &[SupportedStreamConfigRange]) -> Option<&SupportedStreamConfigRange> {
|
||||||
|
ranges
|
||||||
|
.iter()
|
||||||
|
.filter(|r| {
|
||||||
|
r.sample_format() == SampleFormat::F32
|
||||||
|
&& r.min_sample_rate() <= OPUS_RATE
|
||||||
|
&& r.max_sample_rate() >= OPUS_RATE
|
||||||
|
})
|
||||||
|
.min_by_key(|r| (r.channels() != 1, r.channels()))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn frame_samples(sample_rate: u32) -> usize {
|
pub fn frame_samples(sample_rate: u32) -> usize {
|
||||||
(sample_rate as usize * FRAME_MS as usize) / 1000
|
(sample_rate as usize * FRAME_MS as usize) / 1000
|
||||||
}
|
}
|
||||||
|
|
@ -118,7 +131,47 @@ pub fn resample_rate(input: &[f32], from_hz: u32, to_hz: u32) -> Vec<f32> {
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting};
|
use super::{
|
||||||
|
OPUS_RATE, SampleFormat, SupportedStreamConfigRange, best_opus_range,
|
||||||
|
bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting,
|
||||||
|
};
|
||||||
|
|
||||||
|
use cpal::SupportedBufferSize;
|
||||||
|
|
||||||
|
fn range(
|
||||||
|
channels: u16,
|
||||||
|
min: u32,
|
||||||
|
max: u32,
|
||||||
|
format: SampleFormat,
|
||||||
|
) -> SupportedStreamConfigRange {
|
||||||
|
SupportedStreamConfigRange::new(channels, min, max, SupportedBufferSize::Unknown, format)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_prefers_mono_f32_at_the_opus_rate() {
|
||||||
|
let ranges = [
|
||||||
|
range(2, 1, 384_000, SampleFormat::F32),
|
||||||
|
range(1, 1, 384_000, SampleFormat::I16),
|
||||||
|
range(1, 1, 384_000, SampleFormat::F32),
|
||||||
|
];
|
||||||
|
let best = best_opus_range(&ranges).expect("f32 range");
|
||||||
|
assert_eq!(
|
||||||
|
(best.channels(), best.sample_format()),
|
||||||
|
(1, SampleFormat::F32)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_falls_back_to_stereo_and_rejects_non_f32_or_wrong_rate() {
|
||||||
|
let stereo = [range(2, 8_000, 96_000, SampleFormat::F32)];
|
||||||
|
assert_eq!(best_opus_range(&stereo).map(|r| r.channels()), Some(2));
|
||||||
|
let unusable = [
|
||||||
|
range(1, 8_000, 44_100, SampleFormat::F32),
|
||||||
|
range(1, 1, 96_000, SampleFormat::I16),
|
||||||
|
];
|
||||||
|
assert!(best_opus_range(&unusable).is_none());
|
||||||
|
assert_eq!(OPUS_RATE, 48_000);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn jitter_size_maps_to_ms() {
|
fn jitter_size_maps_to_ms() {
|
||||||
|
|
|
||||||
201
src/audio/diagnostics.rs
Normal file
201
src/audio/diagnostics.rs
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
//! Shared audio health: failures raised on capture and playback threads must reach the UI.
|
||||||
|
//!
|
||||||
|
//! Earlier versions only logged such failures (`tracing::error!` on a detached thread), so a
|
||||||
|
//! microphone that could not be opened still showed a connected voice room with no sound. The
|
||||||
|
//! threads now report into [`Diagnostics`]; the UI polls it and shows the message.
|
||||||
|
|
||||||
|
use std::sync::{
|
||||||
|
Mutex,
|
||||||
|
atomic::{AtomicU32, AtomicU64, Ordering},
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Which half of the audio pipeline raised an issue.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum AudioStage {
|
||||||
|
Capture,
|
||||||
|
Playback,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One reportable audio problem.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct AudioIssue {
|
||||||
|
pub stage: AudioStage,
|
||||||
|
pub message: String,
|
||||||
|
/// The stage cannot continue (stream not built, thread ended).
|
||||||
|
pub fatal: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Counters for support logs: where in the chain did the signal stop?
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq)]
|
||||||
|
pub struct AudioStats {
|
||||||
|
pub captured_frames: u64,
|
||||||
|
pub transmitted_frames: u64,
|
||||||
|
pub received_packets: u64,
|
||||||
|
pub decoded_frames: u64,
|
||||||
|
/// Loudest input sample seen since the last snapshot, 0.0 to 1.0.
|
||||||
|
pub input_peak: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Failures {
|
||||||
|
capture: bool,
|
||||||
|
playback: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Thread-safe sink for issues and counters, shared by the pipeline threads.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct Diagnostics {
|
||||||
|
issues: Mutex<Vec<AudioIssue>>,
|
||||||
|
failed: Mutex<Failures>,
|
||||||
|
last_message: Mutex<Option<(AudioStage, String)>>,
|
||||||
|
captured_frames: AtomicU64,
|
||||||
|
transmitted_frames: AtomicU64,
|
||||||
|
received_packets: AtomicU64,
|
||||||
|
decoded_frames: AtomicU64,
|
||||||
|
input_peak_bits: AtomicU32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Diagnostics {
|
||||||
|
/// Records an issue (and logs it). Identical consecutive reports from one stage are
|
||||||
|
/// collapsed so a failing device callback cannot flood the UI.
|
||||||
|
pub fn report(&self, stage: AudioStage, fatal: bool, message: impl Into<String>) {
|
||||||
|
let message = message.into();
|
||||||
|
if fatal && let Ok(mut failed) = self.failed.lock() {
|
||||||
|
match stage {
|
||||||
|
AudioStage::Capture => failed.capture = true,
|
||||||
|
AudioStage::Playback => failed.playback = true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Ok(mut last) = self.last_message.lock() {
|
||||||
|
if last
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|(s, m)| *s == stage && *m == message)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
*last = Some((stage, message.clone()));
|
||||||
|
}
|
||||||
|
tracing::error!(?stage, fatal, "audio: {message}");
|
||||||
|
if let Ok(mut issues) = self.issues.lock() {
|
||||||
|
if issues.len() >= 16 {
|
||||||
|
issues.remove(0);
|
||||||
|
}
|
||||||
|
issues.push(AudioIssue {
|
||||||
|
stage,
|
||||||
|
message,
|
||||||
|
fatal,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drains the issues reported since the last call.
|
||||||
|
pub fn take_issues(&self) -> Vec<AudioIssue> {
|
||||||
|
self.issues
|
||||||
|
.lock()
|
||||||
|
.map(|mut issues| std::mem::take(&mut *issues))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True when neither capture nor playback is working any more.
|
||||||
|
pub fn all_failed(&self) -> bool {
|
||||||
|
self.failed
|
||||||
|
.lock()
|
||||||
|
.map(|failed| failed.capture && failed.playback)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn count_captured(&self, peak: f32) {
|
||||||
|
self.captured_frames.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let mut current = self.input_peak_bits.load(Ordering::Relaxed);
|
||||||
|
while peak > f32::from_bits(current) {
|
||||||
|
match self.input_peak_bits.compare_exchange_weak(
|
||||||
|
current,
|
||||||
|
peak.to_bits(),
|
||||||
|
Ordering::Relaxed,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
) {
|
||||||
|
Ok(_) => break,
|
||||||
|
Err(actual) => current = actual,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn count_transmitted(&self) {
|
||||||
|
self.transmitted_frames.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn count_received(&self) {
|
||||||
|
self.received_packets.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn count_decoded(&self) {
|
||||||
|
self.decoded_frames.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Current counters; the input peak is reset so each snapshot covers one interval.
|
||||||
|
pub fn snapshot(&self) -> AudioStats {
|
||||||
|
AudioStats {
|
||||||
|
captured_frames: self.captured_frames.load(Ordering::Relaxed),
|
||||||
|
transmitted_frames: self.transmitted_frames.load(Ordering::Relaxed),
|
||||||
|
received_packets: self.received_packets.load(Ordering::Relaxed),
|
||||||
|
decoded_frames: self.decoded_frames.load(Ordering::Relaxed),
|
||||||
|
input_peak: f32::from_bits(self.input_peak_bits.swap(0, Ordering::Relaxed)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn issues_are_delivered_once_and_duplicates_collapse() {
|
||||||
|
let diag = Diagnostics::default();
|
||||||
|
diag.report(AudioStage::Capture, true, "no microphone");
|
||||||
|
diag.report(AudioStage::Capture, true, "no microphone");
|
||||||
|
diag.report(AudioStage::Playback, false, "xrun");
|
||||||
|
let issues = diag.take_issues();
|
||||||
|
assert_eq!(issues.len(), 2);
|
||||||
|
assert_eq!(issues[0].stage, AudioStage::Capture);
|
||||||
|
assert!(issues[0].fatal);
|
||||||
|
assert!(diag.take_issues().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn all_failed_requires_both_stages_to_fail_fatally() {
|
||||||
|
let diag = Diagnostics::default();
|
||||||
|
diag.report(AudioStage::Capture, true, "a");
|
||||||
|
assert!(!diag.all_failed());
|
||||||
|
diag.report(AudioStage::Playback, false, "b");
|
||||||
|
assert!(!diag.all_failed());
|
||||||
|
diag.report(AudioStage::Playback, true, "c");
|
||||||
|
assert!(diag.all_failed());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn snapshot_reports_counters_and_resets_the_peak() {
|
||||||
|
let diag = Diagnostics::default();
|
||||||
|
diag.count_captured(0.2);
|
||||||
|
diag.count_captured(0.5);
|
||||||
|
diag.count_captured(0.1);
|
||||||
|
diag.count_transmitted();
|
||||||
|
diag.count_received();
|
||||||
|
diag.count_decoded();
|
||||||
|
let first = diag.snapshot();
|
||||||
|
assert_eq!(first.captured_frames, 3);
|
||||||
|
assert_eq!(first.transmitted_frames, 1);
|
||||||
|
assert_eq!(first.received_packets, 1);
|
||||||
|
assert_eq!(first.decoded_frames, 1);
|
||||||
|
assert!((first.input_peak - 0.5).abs() < f32::EPSILON);
|
||||||
|
assert_eq!(diag.snapshot().input_peak, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn issue_queue_is_bounded() {
|
||||||
|
let diag = Diagnostics::default();
|
||||||
|
for i in 0..40 {
|
||||||
|
diag.report(AudioStage::Playback, false, format!("e{i}"));
|
||||||
|
}
|
||||||
|
assert_eq!(diag.take_issues().len(), 16);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,12 +1,14 @@
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod devices;
|
pub mod devices;
|
||||||
|
pub mod diagnostics;
|
||||||
mod pipeline;
|
mod pipeline;
|
||||||
mod processing;
|
mod processing;
|
||||||
|
|
||||||
pub use devices::{DeviceEntry, DeviceLists};
|
pub use devices::{DeviceEntry, DeviceLists};
|
||||||
|
pub use diagnostics::{AudioIssue, AudioStage, AudioStats};
|
||||||
pub use pipeline::JitterConfig;
|
pub use pipeline::JitterConfig;
|
||||||
pub use pipeline::{AudioPipeline, MediaMixer};
|
pub use pipeline::{AudioPipeline, MediaMixer};
|
||||||
pub use processing::{DenoiseState, TransmitGate};
|
pub use processing::{DenoiseState, TransmitGate, VoiceActivity};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
pub struct AudioSync {
|
pub struct AudioSync {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
use anyhow::Result;
|
use anyhow::{Context, Result};
|
||||||
use cpal::traits::{DeviceTrait, StreamTrait};
|
use cpal::traits::{DeviceTrait, StreamTrait};
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
@ -7,9 +7,13 @@ use tracing::{info, warn};
|
||||||
use crate::audio::{
|
use crate::audio::{
|
||||||
EncodedFrame,
|
EncodedFrame,
|
||||||
config::{self, AudioConfig, OPUS_RATE},
|
config::{self, AudioConfig, OPUS_RATE},
|
||||||
processing::{DenoiseState, TransmitGate},
|
diagnostics::{AudioStage, Diagnostics},
|
||||||
|
processing::{DenoiseState, TransmitGate, VoiceActivity},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// Seconds of exact digital silence after which the microphone is reported as dead.
|
||||||
|
const SILENCE_WARNING_SECS: u32 = 4;
|
||||||
|
|
||||||
const MAX_OPUS: usize = 4000;
|
const MAX_OPUS: usize = 4000;
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
|
@ -24,6 +28,7 @@ pub fn run(
|
||||||
local_speaking: Arc<AtomicBool>,
|
local_speaking: Arc<AtomicBool>,
|
||||||
stopped: Arc<AtomicBool>,
|
stopped: Arc<AtomicBool>,
|
||||||
microphone_tap: super::media::MicrophoneTap,
|
microphone_tap: super::media::MicrophoneTap,
|
||||||
|
diagnostics: Arc<Diagnostics>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let mut encoder = opus::Encoder::new(OPUS_RATE, opus::Channels::Mono, opus::Application::Voip)?;
|
let mut encoder = opus::Encoder::new(OPUS_RATE, opus::Channels::Mono, opus::Application::Voip)?;
|
||||||
let mut last_bitrate_bps: u32 = 0;
|
let mut last_bitrate_bps: u32 = 0;
|
||||||
|
|
@ -41,24 +46,38 @@ pub fn run(
|
||||||
let mut voice_seq: u32 = 0;
|
let mut voice_seq: u32 = 0;
|
||||||
let mut timestamp: u32 = 0;
|
let mut timestamp: u32 = 0;
|
||||||
|
|
||||||
let stream = device.build_input_stream(
|
let error_diagnostics = diagnostics.clone();
|
||||||
cfg.stream,
|
let stream = device
|
||||||
move |data: &[f32], _| {
|
.build_input_stream(
|
||||||
let mut b = buf2.lock().unwrap();
|
cfg.stream,
|
||||||
if in_channels <= 1 {
|
move |data: &[f32], _| {
|
||||||
b.extend_from_slice(data);
|
let mut b = buf2.lock().unwrap();
|
||||||
} else {
|
if in_channels <= 1 {
|
||||||
for frame in data.chunks(in_channels) {
|
b.extend_from_slice(data);
|
||||||
let sum: f32 = frame.iter().sum();
|
} else {
|
||||||
b.push(sum / in_channels as f32);
|
for frame in data.chunks(in_channels) {
|
||||||
|
let sum: f32 = frame.iter().sum();
|
||||||
|
b.push(sum / in_channels as f32);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
},
|
move |e| {
|
||||||
|e| tracing::error!("input: {e}"),
|
error_diagnostics.report(
|
||||||
None,
|
AudioStage::Capture,
|
||||||
)?;
|
false,
|
||||||
stream.play()?;
|
format!("microphone stream: {e}"),
|
||||||
|
)
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.context("cannot open the microphone stream")?;
|
||||||
|
stream
|
||||||
|
.play()
|
||||||
|
.context("cannot start the microphone stream")?;
|
||||||
info!("capture started ({} Hz)", cfg.sample_rate);
|
info!("capture started ({} Hz)", cfg.sample_rate);
|
||||||
|
let mut vad = VoiceActivity::default();
|
||||||
|
let mut silent_frames: u32 = 0;
|
||||||
|
let mut silence_reported = false;
|
||||||
|
|
||||||
while !stopped.load(Ordering::Acquire) {
|
while !stopped.load(Ordering::Acquire) {
|
||||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||||
|
|
@ -80,12 +99,29 @@ pub fn run(
|
||||||
padded
|
padded
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let peak = raw_peak(&pcm);
|
||||||
|
diagnostics.count_captured(peak);
|
||||||
|
if peak == 0.0 {
|
||||||
|
silent_frames = silent_frames.saturating_add(1);
|
||||||
|
if !silence_reported && silent_frames * frame_ms >= SILENCE_WARNING_SECS * 1000 {
|
||||||
|
silence_reported = true;
|
||||||
|
diagnostics.report(
|
||||||
|
AudioStage::Capture,
|
||||||
|
false,
|
||||||
|
"the microphone delivers only silence; check the input device and its mute switch",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
silent_frames = 0;
|
||||||
|
silence_reported = false;
|
||||||
|
}
|
||||||
|
|
||||||
let gate_snapshot = gate.lock().unwrap().clone();
|
let gate_snapshot = gate.lock().unwrap().clone();
|
||||||
TransmitGate::apply_input_volume(&mut pcm, gate_snapshot.input_volume);
|
TransmitGate::apply_input_volume(&mut pcm, gate_snapshot.input_volume);
|
||||||
if gate_snapshot.noise_suppress {
|
if gate_snapshot.noise_suppress {
|
||||||
denoise.process(&mut pcm);
|
denoise.process(&mut pcm);
|
||||||
}
|
}
|
||||||
let speaking = TransmitGate::should_transmit(&gate_snapshot, &pcm);
|
let speaking = gate_snapshot.decide(&pcm, &mut vad, frame_ms);
|
||||||
local_speaking.store(speaking, Ordering::Relaxed);
|
local_speaking.store(speaking, Ordering::Relaxed);
|
||||||
if !speaking {
|
if !speaking {
|
||||||
b = buf.lock().unwrap();
|
b = buf.lock().unwrap();
|
||||||
|
|
@ -110,6 +146,7 @@ pub fn run(
|
||||||
{
|
{
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
diagnostics.count_transmitted();
|
||||||
voice_seq = voice_seq.wrapping_add(1);
|
voice_seq = voice_seq.wrapping_add(1);
|
||||||
timestamp = timestamp.wrapping_add(opus_frame as u32);
|
timestamp = timestamp.wrapping_add(opus_frame as u32);
|
||||||
}
|
}
|
||||||
|
|
@ -121,6 +158,10 @@ pub fn run(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn raw_peak(samples: &[f32]) -> f32 {
|
||||||
|
samples.iter().fold(0.0f32, |peak, s| peak.max(s.abs()))
|
||||||
|
}
|
||||||
|
|
||||||
fn frame_samples_at(sample_rate: u32, frame_ms: u32) -> usize {
|
fn frame_samples_at(sample_rate: u32, frame_ms: u32) -> usize {
|
||||||
(sample_rate as usize * frame_ms as usize) / 1000
|
(sample_rate as usize * frame_ms as usize) / 1000
|
||||||
}
|
}
|
||||||
|
|
|
||||||
113
src/audio/pipeline/effects.rs
Normal file
113
src/audio/pipeline/effects.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
//! Short notification sounds mixed into the already open playback stream.
|
||||||
|
//!
|
||||||
|
//! No extra device is opened while a voice room is active, so effects follow the selected
|
||||||
|
//! output device and cannot conflict with exclusive hardware access. The audio callback only
|
||||||
|
//! pops samples with `try_lock`; it never waits for the UI thread.
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
collections::VecDeque,
|
||||||
|
sync::{Arc, Mutex},
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::audio::config::{OPUS_RATE, resample_rate};
|
||||||
|
|
||||||
|
/// Longest backlog of queued effect audio (two seconds) so a burst cannot grow without bound.
|
||||||
|
const MAX_SECONDS: usize = 2;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct EffectQueue {
|
||||||
|
samples: Arc<Mutex<VecDeque<f32>>>,
|
||||||
|
device_rate: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EffectQueue {
|
||||||
|
pub(super) fn new(device_rate: u32) -> Self {
|
||||||
|
Self {
|
||||||
|
samples: Arc::default(),
|
||||||
|
device_rate,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mixes a mono 48 kHz buffer (volume already applied) on top of what is still queued.
|
||||||
|
pub fn push(&self, mono_48k: &[f32]) {
|
||||||
|
let converted = if self.device_rate == OPUS_RATE {
|
||||||
|
mono_48k.to_vec()
|
||||||
|
} else {
|
||||||
|
resample_rate(mono_48k, OPUS_RATE, self.device_rate)
|
||||||
|
};
|
||||||
|
let limit = self.device_rate as usize * MAX_SECONDS;
|
||||||
|
let Ok(mut queue) = self.samples.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for (index, sample) in converted.into_iter().take(limit).enumerate() {
|
||||||
|
if let Some(existing) = queue.get_mut(index) {
|
||||||
|
*existing += sample;
|
||||||
|
} else {
|
||||||
|
queue.push_back(sample);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Adds queued effect samples to an interleaved output buffer.
|
||||||
|
pub(super) fn mix_into(&self, out: &mut [f32], channels: usize) {
|
||||||
|
let Ok(mut queue) = self.samples.try_lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if queue.is_empty() || channels == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for frame in out.chunks_mut(channels) {
|
||||||
|
let Some(sample) = queue.pop_front() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
for value in frame {
|
||||||
|
*value = (*value + sample).clamp(-1.0, 1.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn clear(&self) {
|
||||||
|
if let Ok(mut queue) = self.samples.lock() {
|
||||||
|
queue.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn effects_play_once_on_every_channel_and_then_stop() {
|
||||||
|
let queue = EffectQueue::new(48_000);
|
||||||
|
queue.push(&[0.5, 0.25]);
|
||||||
|
let mut out = [0.0f32; 8];
|
||||||
|
queue.mix_into(&mut out, 2);
|
||||||
|
assert_eq!(out, [0.5, 0.5, 0.25, 0.25, 0.0, 0.0, 0.0, 0.0]);
|
||||||
|
let mut again = [0.0f32; 4];
|
||||||
|
queue.mix_into(&mut again, 2);
|
||||||
|
assert_eq!(again, [0.0; 4]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn overlapping_effects_add_and_the_output_is_clamped() {
|
||||||
|
let queue = EffectQueue::new(48_000);
|
||||||
|
queue.push(&[0.4, 0.4]);
|
||||||
|
queue.push(&[0.4, 0.4, 0.4]);
|
||||||
|
let mut out = [0.9f32; 3];
|
||||||
|
queue.mix_into(&mut out, 1);
|
||||||
|
assert_eq!(out, [1.0, 1.0, 1.0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn queue_is_bounded_and_resampled_to_the_device_rate() {
|
||||||
|
let queue = EffectQueue::new(24_000);
|
||||||
|
queue.push(&vec![0.1; 48_000]);
|
||||||
|
let length = queue.samples.lock().unwrap().len();
|
||||||
|
assert_eq!(length, 24_000);
|
||||||
|
queue.clear();
|
||||||
|
assert!(queue.samples.lock().unwrap().is_empty());
|
||||||
|
queue.push(&vec![0.1; 48_000 * 10]);
|
||||||
|
assert_eq!(queue.samples.lock().unwrap().len(), 24_000 * MAX_SECONDS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
mod capture;
|
mod capture;
|
||||||
|
mod effects;
|
||||||
mod media;
|
mod media;
|
||||||
pub use media::MediaMixer;
|
pub use media::MediaMixer;
|
||||||
mod playback;
|
mod playback;
|
||||||
|
|
@ -12,6 +13,7 @@ use tracing::info;
|
||||||
|
|
||||||
use crate::audio::{
|
use crate::audio::{
|
||||||
AudioSync, EncodedFrame, IncomingVoice, PipelineConfig, config, devices,
|
AudioSync, EncodedFrame, IncomingVoice, PipelineConfig, config, devices,
|
||||||
|
diagnostics::{AudioIssue, AudioStage, AudioStats, Diagnostics},
|
||||||
processing::TransmitGate,
|
processing::TransmitGate,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -22,6 +24,8 @@ pub struct AudioPipeline {
|
||||||
encoded_rx: Receiver<EncodedFrame>,
|
encoded_rx: Receiver<EncodedFrame>,
|
||||||
microphone_tap: media::MicrophoneTap,
|
microphone_tap: media::MicrophoneTap,
|
||||||
media_mixer: MediaMixer,
|
media_mixer: MediaMixer,
|
||||||
|
effects: effects::EffectQueue,
|
||||||
|
diagnostics: Arc<Diagnostics>,
|
||||||
voice_tx: Sender<IncomingVoice>,
|
voice_tx: Sender<IncomingVoice>,
|
||||||
_encoded_tx: Sender<EncodedFrame>,
|
_encoded_tx: Sender<EncodedFrame>,
|
||||||
gate: Arc<Mutex<TransmitGate>>,
|
gate: Arc<Mutex<TransmitGate>>,
|
||||||
|
|
@ -40,6 +44,7 @@ impl Drop for AudioPipeline {
|
||||||
self.stopped.store(true, Ordering::Release);
|
self.stopped.store(true, Ordering::Release);
|
||||||
self.microphone_tap.enable(false);
|
self.microphone_tap.enable(false);
|
||||||
self.media_mixer.clear();
|
self.media_mixer.clear();
|
||||||
|
self.effects.clear();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -54,6 +59,27 @@ impl AudioPipeline {
|
||||||
self.media_mixer.clone()
|
self.media_mixer.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Plays a short notification (mono, 48 kHz, volume already applied) through the open
|
||||||
|
/// playback stream without touching the voice path.
|
||||||
|
pub fn play_effect(&self, mono_48k: &[f32]) {
|
||||||
|
self.effects.push(mono_48k);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Problems raised by the capture and playback threads since the last call.
|
||||||
|
pub fn take_issues(&self) -> Vec<AudioIssue> {
|
||||||
|
self.diagnostics.take_issues()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True when neither the microphone nor the output could be used.
|
||||||
|
pub fn is_dead(&self) -> bool {
|
||||||
|
self.diagnostics.all_failed()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Signal counters for support logs.
|
||||||
|
pub fn stats(&self) -> AudioStats {
|
||||||
|
self.diagnostics.snapshot()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn poll_encoded(&self) -> Option<EncodedFrame> {
|
pub fn poll_encoded(&self) -> Option<EncodedFrame> {
|
||||||
self.encoded_rx.try_recv().ok()
|
self.encoded_rx.try_recv().ok()
|
||||||
}
|
}
|
||||||
|
|
@ -124,6 +150,11 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result<AudioPipeline> {
|
||||||
let tap_capture = microphone_tap.clone();
|
let tap_capture = microphone_tap.clone();
|
||||||
let media_mixer = MediaMixer::new(out_cfg.sample_rate);
|
let media_mixer = MediaMixer::new(out_cfg.sample_rate);
|
||||||
let mixer_playback = media_mixer.clone();
|
let mixer_playback = media_mixer.clone();
|
||||||
|
let effects = effects::EffectQueue::new(out_cfg.sample_rate);
|
||||||
|
let effects_playback = effects.clone();
|
||||||
|
let diagnostics = Arc::new(Diagnostics::default());
|
||||||
|
let diagnostics_capture = diagnostics.clone();
|
||||||
|
let diagnostics_playback = diagnostics.clone();
|
||||||
|
|
||||||
let (encoded_tx, encoded_rx) = mpsc::channel::<EncodedFrame>();
|
let (encoded_tx, encoded_rx) = mpsc::channel::<EncodedFrame>();
|
||||||
let (voice_tx, voice_rx) = mpsc::channel::<IncomingVoice>();
|
let (voice_tx, voice_rx) = mpsc::channel::<IncomingVoice>();
|
||||||
|
|
@ -162,8 +193,9 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result<AudioPipeline> {
|
||||||
local_speaking_cap,
|
local_speaking_cap,
|
||||||
capture_stopped,
|
capture_stopped,
|
||||||
tap_capture,
|
tap_capture,
|
||||||
|
diagnostics_capture.clone(),
|
||||||
) {
|
) {
|
||||||
tracing::error!("capture: {e}");
|
diagnostics_capture.report(AudioStage::Capture, true, format!("{e:#}"));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
|
|
@ -175,8 +207,10 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result<AudioPipeline> {
|
||||||
jitter_play,
|
jitter_play,
|
||||||
last_remote_voice_play,
|
last_remote_voice_play,
|
||||||
mixer_playback,
|
mixer_playback,
|
||||||
|
effects_playback,
|
||||||
|
diagnostics_playback.clone(),
|
||||||
) {
|
) {
|
||||||
tracing::error!("playback: {e}");
|
diagnostics_playback.report(AudioStage::Playback, true, format!("{e:#}"));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -185,6 +219,8 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result<AudioPipeline> {
|
||||||
encoded_rx,
|
encoded_rx,
|
||||||
microphone_tap,
|
microphone_tap,
|
||||||
media_mixer,
|
media_mixer,
|
||||||
|
effects,
|
||||||
|
diagnostics,
|
||||||
voice_tx,
|
voice_tx,
|
||||||
_encoded_tx: encoded_tx,
|
_encoded_tx: encoded_tx,
|
||||||
gate,
|
gate,
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
use crate::jitter::{BufferedPacket, JitterBuffer};
|
use crate::jitter::{BufferedPacket, JitterBuffer};
|
||||||
use anyhow::Result;
|
use anyhow::{Context, Result};
|
||||||
use cpal::traits::{DeviceTrait, StreamTrait};
|
use cpal::traits::{DeviceTrait, StreamTrait};
|
||||||
use std::sync::{
|
use std::sync::{
|
||||||
Arc, Mutex,
|
Arc, Mutex,
|
||||||
|
|
@ -11,6 +11,7 @@ use tracing::{info, warn};
|
||||||
|
|
||||||
use crate::audio::IncomingVoice;
|
use crate::audio::IncomingVoice;
|
||||||
use crate::audio::config::{self, AudioConfig, OPUS_RATE};
|
use crate::audio::config::{self, AudioConfig, OPUS_RATE};
|
||||||
|
use crate::audio::diagnostics::{AudioStage, Diagnostics};
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct JitterConfig {
|
pub struct JitterConfig {
|
||||||
|
|
@ -27,6 +28,8 @@ pub fn run(
|
||||||
jitter_cfg: Arc<Mutex<JitterConfig>>,
|
jitter_cfg: Arc<Mutex<JitterConfig>>,
|
||||||
last_remote_voice_ms: Arc<AtomicU64>,
|
last_remote_voice_ms: Arc<AtomicU64>,
|
||||||
media_mixer: super::media::MediaMixer,
|
media_mixer: super::media::MediaMixer,
|
||||||
|
effects: super::effects::EffectQueue,
|
||||||
|
diagnostics: Arc<Diagnostics>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let mut decoder = opus::Decoder::new(OPUS_RATE, opus::Channels::Mono)?;
|
let mut decoder = opus::Decoder::new(OPUS_RATE, opus::Channels::Mono)?;
|
||||||
let pb: Arc<Mutex<Vec<f32>>> = Arc::new(Mutex::new(Vec::new()));
|
let pb: Arc<Mutex<Vec<f32>>> = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
|
@ -37,51 +40,60 @@ pub fn run(
|
||||||
let opus_frame = config::frame_samples(OPUS_RATE);
|
let opus_frame = config::frame_samples(OPUS_RATE);
|
||||||
|
|
||||||
let media_queues = media_mixer.queues.clone();
|
let media_queues = media_mixer.queues.clone();
|
||||||
let stream = device.build_output_stream(
|
let error_diagnostics = diagnostics.clone();
|
||||||
cfg.stream,
|
let effects_playback = effects.clone();
|
||||||
move |out: &mut [f32], _| {
|
let stream = device
|
||||||
let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0);
|
.build_output_stream(
|
||||||
let mut buf = pb2.lock().unwrap();
|
cfg.stream,
|
||||||
if out_channels == 1 {
|
move |out: &mut [f32], _| {
|
||||||
let n = buf.len().min(out.len());
|
let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0);
|
||||||
for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) {
|
let mut buf = pb2.lock().unwrap();
|
||||||
*o = s * gain;
|
if out_channels == 1 {
|
||||||
|
let n = buf.len().min(out.len());
|
||||||
|
for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) {
|
||||||
|
*o = s * gain;
|
||||||
|
}
|
||||||
|
buf.drain(..n);
|
||||||
|
out[n..].fill(0.0);
|
||||||
|
} else {
|
||||||
|
let frames = out.len() / out_channels;
|
||||||
|
for i in 0..frames {
|
||||||
|
let sample = if i < buf.len() { buf[i] * gain } else { 0.0 };
|
||||||
|
for ch in 0..out_channels {
|
||||||
|
out[i * out_channels + ch] = sample;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let drain_n = frames.min(buf.len());
|
||||||
|
buf.drain(..drain_n);
|
||||||
}
|
}
|
||||||
buf.drain(..n);
|
effects_playback.mix_into(out, out_channels);
|
||||||
out[n..].fill(0.0);
|
if let Ok(mut queues) = media_queues.lock() {
|
||||||
} else {
|
for frame in out.chunks_mut(out_channels) {
|
||||||
let frames = out.len() / out_channels;
|
let mut mixed = [0.0f32; 2];
|
||||||
for i in 0..frames {
|
for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) {
|
||||||
let sample = if i < buf.len() { buf[i] * gain } else { 0.0 };
|
mixed[0] += samples[0];
|
||||||
for ch in 0..out_channels {
|
mixed[1] += samples[1];
|
||||||
out[i * out_channels + ch] = sample;
|
}
|
||||||
|
for (channel, sample) in frame.iter_mut().enumerate() {
|
||||||
|
let value = if out_channels == 1 || channel > 1 {
|
||||||
|
(mixed[0] + mixed[1]) * 0.5
|
||||||
|
} else {
|
||||||
|
mixed[channel]
|
||||||
|
};
|
||||||
|
*sample = (*sample + value * gain).clamp(-1.0, 1.0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let drain_n = frames.min(buf.len());
|
},
|
||||||
buf.drain(..drain_n);
|
move |e| {
|
||||||
}
|
error_diagnostics.report(AudioStage::Playback, false, format!("output stream: {e}"))
|
||||||
if let Ok(mut queues) = media_queues.lock() {
|
},
|
||||||
for frame in out.chunks_mut(out_channels) {
|
None,
|
||||||
let mut mixed = [0.0f32; 2];
|
)
|
||||||
for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) {
|
.context("cannot open the audio output stream")?;
|
||||||
mixed[0] += samples[0];
|
stream
|
||||||
mixed[1] += samples[1];
|
.play()
|
||||||
}
|
.context("cannot start the audio output stream")?;
|
||||||
for (channel, sample) in frame.iter_mut().enumerate() {
|
|
||||||
let value = if out_channels == 1 || channel > 1 {
|
|
||||||
(mixed[0] + mixed[1]) * 0.5
|
|
||||||
} else {
|
|
||||||
mixed[channel]
|
|
||||||
};
|
|
||||||
*sample = (*sample + value * gain).clamp(-1.0, 1.0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|e| tracing::error!("output: {e}"),
|
|
||||||
None,
|
|
||||||
)?;
|
|
||||||
stream.play()?;
|
|
||||||
info!("playback started ({} Hz)", cfg.sample_rate);
|
info!("playback started ({} Hz)", cfg.sample_rate);
|
||||||
|
|
||||||
let clock = Instant::now();
|
let clock = Instant::now();
|
||||||
|
|
@ -121,6 +133,7 @@ pub fn run(
|
||||||
opus_data: pkt.opus_data,
|
opus_data: pkt.opus_data,
|
||||||
arrived_at: clock.elapsed().as_millis() as u64,
|
arrived_at: clock.elapsed().as_millis() as u64,
|
||||||
});
|
});
|
||||||
|
diagnostics.count_received();
|
||||||
}
|
}
|
||||||
Err(TryRecvError::Empty) => break,
|
Err(TryRecvError::Empty) => break,
|
||||||
Err(TryRecvError::Disconnected) => {
|
Err(TryRecvError::Disconnected) => {
|
||||||
|
|
@ -138,6 +151,7 @@ pub fn run(
|
||||||
needs_resample,
|
needs_resample,
|
||||||
cfg.sample_rate,
|
cfg.sample_rate,
|
||||||
&pb,
|
&pb,
|
||||||
|
&diagnostics,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
std::thread::sleep(Duration::from_millis(2));
|
std::thread::sleep(Duration::from_millis(2));
|
||||||
|
|
@ -152,6 +166,7 @@ fn decode_into_playback(
|
||||||
needs_resample: bool,
|
needs_resample: bool,
|
||||||
device_rate: u32,
|
device_rate: u32,
|
||||||
pb: &Arc<Mutex<Vec<f32>>>,
|
pb: &Arc<Mutex<Vec<f32>>>,
|
||||||
|
diagnostics: &Diagnostics,
|
||||||
) {
|
) {
|
||||||
let mut pcm = vec![0f32; opus_frame];
|
let mut pcm = vec![0f32; opus_frame];
|
||||||
match decoder.decode_float(data, &mut pcm, false) {
|
match decoder.decode_float(data, &mut pcm, false) {
|
||||||
|
|
@ -163,6 +178,7 @@ fn decode_into_playback(
|
||||||
pcm
|
pcm
|
||||||
};
|
};
|
||||||
pb.lock().unwrap().extend_from_slice(&samples);
|
pb.lock().unwrap().extend_from_slice(&samples);
|
||||||
|
diagnostics.count_decoded();
|
||||||
}
|
}
|
||||||
Err(e) => warn!("decode: {e}"),
|
Err(e) => warn!("decode: {e}"),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,8 +15,8 @@ impl Default for TransmitGate {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
mic_enabled: true,
|
mic_enabled: true,
|
||||||
vad_mode: 0,
|
vad_mode: 1,
|
||||||
vad_threshold: 40.0,
|
vad_threshold: 15.0,
|
||||||
ptt_held: false,
|
ptt_held: false,
|
||||||
input_volume: 80.0,
|
input_volume: 80.0,
|
||||||
noise_suppress: false,
|
noise_suppress: false,
|
||||||
|
|
@ -62,6 +62,59 @@ fn vad_rms_threshold(ui_threshold: f32) -> f32 {
|
||||||
0.002 + (ui_threshold.clamp(0.0, 100.0) / 100.0) * 0.118
|
0.002 + (ui_threshold.clamp(0.0, 100.0) / 100.0) * 0.118
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Lowest RMS the automatic threshold may settle on (about -48 dBFS).
|
||||||
|
const VAD_MIN_THRESHOLD: f32 = 0.004;
|
||||||
|
/// How long transmission continues after the signal drops below the threshold.
|
||||||
|
const VAD_HANGOVER_MS: u32 = 300;
|
||||||
|
|
||||||
|
/// Voice activity detector with an automatically tracked noise floor.
|
||||||
|
///
|
||||||
|
/// The user threshold is only a ceiling: the detector opens a little above the measured
|
||||||
|
/// background noise, so quiet microphones work without manual tuning and noisy rooms do not
|
||||||
|
/// keep the channel open. A short hangover avoids clipping word endings.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct VoiceActivity {
|
||||||
|
noise_floor: f32,
|
||||||
|
hangover_ms: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for VoiceActivity {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
noise_floor: 0.004,
|
||||||
|
hangover_ms: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VoiceActivity {
|
||||||
|
/// Current opening threshold for the given ceiling.
|
||||||
|
pub fn threshold(&self, ceiling: f32) -> f32 {
|
||||||
|
(self.noise_floor * 3.0 + 0.002)
|
||||||
|
.max(VAD_MIN_THRESHOLD)
|
||||||
|
.min(ceiling.max(VAD_MIN_THRESHOLD))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Feeds one frame (`frame_ms` long) and tells whether it should be transmitted.
|
||||||
|
pub fn update(&mut self, rms: f32, ceiling: f32, frame_ms: u32) -> bool {
|
||||||
|
let threshold = self.threshold(ceiling);
|
||||||
|
if rms >= threshold {
|
||||||
|
self.hangover_ms = VAD_HANGOVER_MS;
|
||||||
|
// Sustained sound the detector cannot tell from speech slowly raises the floor, so a
|
||||||
|
// fan that starts mid-call stops holding the channel open.
|
||||||
|
self.noise_floor += (rms.min(ceiling / 3.0) - self.noise_floor).max(0.0) * 0.0005;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let rate = if rms < self.noise_floor { 0.2 } else { 0.02 };
|
||||||
|
self.noise_floor += (rms - self.noise_floor) * rate;
|
||||||
|
if self.hangover_ms > 0 {
|
||||||
|
self.hangover_ms = self.hangover_ms.saturating_sub(frame_ms);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl TransmitGate {
|
impl TransmitGate {
|
||||||
pub fn apply_input_volume(pcm: &mut [f32], volume_pct: f32) {
|
pub fn apply_input_volume(pcm: &mut [f32], volume_pct: f32) {
|
||||||
let gain = (volume_pct / 100.0).clamp(0.0, 2.0);
|
let gain = (volume_pct / 100.0).clamp(0.0, 2.0);
|
||||||
|
|
@ -73,6 +126,21 @@ impl TransmitGate {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Like [`should_transmit`](Self::should_transmit) but voice activation uses the adaptive
|
||||||
|
/// detector in `vad` instead of a fixed threshold.
|
||||||
|
pub fn decide(&self, pcm: &[f32], vad: &mut VoiceActivity, frame_ms: u32) -> bool {
|
||||||
|
if self.mic_enabled && self.vad_mode == 1 {
|
||||||
|
// The detector must see every frame to track the noise floor, even when muted
|
||||||
|
// frames are skipped above.
|
||||||
|
return vad.update(
|
||||||
|
frame_rms(pcm),
|
||||||
|
vad_rms_threshold(self.vad_threshold),
|
||||||
|
frame_ms,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.should_transmit(pcm)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn should_transmit(&self, pcm: &[f32]) -> bool {
|
pub fn should_transmit(&self, pcm: &[f32]) -> bool {
|
||||||
if !self.mic_enabled {
|
if !self.mic_enabled {
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -129,6 +197,72 @@ mod tests {
|
||||||
assert!(gate.should_transmit(&loud_frame()));
|
assert!(gate.should_transmit(&loud_frame()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detector_adapts_to_a_quiet_microphone() {
|
||||||
|
let mut vad = VoiceActivity::default();
|
||||||
|
// Room tone at 0.0005 RMS for one second lets the floor settle.
|
||||||
|
for _ in 0..50 {
|
||||||
|
vad.update(0.0005, 0.02, 20);
|
||||||
|
}
|
||||||
|
assert!(vad.threshold(0.02) <= VAD_MIN_THRESHOLD + 1e-6);
|
||||||
|
// Quiet speech at 0.008 RMS would never pass a fixed 0.05 threshold.
|
||||||
|
assert!(vad.update(0.008, 0.02, 20));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detector_never_exceeds_the_user_ceiling() {
|
||||||
|
let mut vad = VoiceActivity::default();
|
||||||
|
for _ in 0..2000 {
|
||||||
|
vad.update(0.5, 0.02, 20);
|
||||||
|
}
|
||||||
|
assert!(vad.threshold(0.02) <= 0.02 + 1e-6);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detector_keeps_transmitting_through_the_hangover_then_stops() {
|
||||||
|
let mut vad = VoiceActivity::default();
|
||||||
|
for _ in 0..20 {
|
||||||
|
vad.update(0.0003, 0.02, 20);
|
||||||
|
}
|
||||||
|
assert!(vad.update(0.05, 0.02, 20));
|
||||||
|
let mut open_frames = 0;
|
||||||
|
while vad.update(0.0003, 0.02, 20) {
|
||||||
|
open_frames += 1;
|
||||||
|
assert!(open_frames < 100);
|
||||||
|
}
|
||||||
|
assert_eq!(open_frames, (VAD_HANGOVER_MS / 20) as usize);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_gate_uses_voice_activation_with_a_sane_ceiling() {
|
||||||
|
let gate = TransmitGate::default();
|
||||||
|
assert_eq!(gate.vad_mode, 1);
|
||||||
|
// The ceiling must be low enough for ordinary speech levels.
|
||||||
|
assert!(vad_rms_threshold(gate.vad_threshold) < 0.03);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_gates_voice_activation_by_mic_state_and_mode() {
|
||||||
|
let mut vad = VoiceActivity::default();
|
||||||
|
let loud = loud_frame();
|
||||||
|
let gate = TransmitGate {
|
||||||
|
vad_mode: 1,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(gate.decide(&loud, &mut vad, 20));
|
||||||
|
let muted = TransmitGate {
|
||||||
|
mic_enabled: false,
|
||||||
|
vad_mode: 1,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(!muted.decide(&loud, &mut vad, 20));
|
||||||
|
let ptt = TransmitGate {
|
||||||
|
vad_mode: 0,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(!ptt.decide(&loud, &mut vad, 20));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn vad_uses_threshold() {
|
fn vad_uses_threshold() {
|
||||||
let gate = TransmitGate {
|
let gate = TransmitGate {
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ impl State {
|
||||||
let events = event_tx.clone();
|
let events = event_tx.clone();
|
||||||
let receiver = tokio::spawn(async move {
|
let receiver = tokio::spawn(async move {
|
||||||
let mut bytes = vec![0u8; voice::MAX_DATAGRAM + 1];
|
let mut bytes = vec![0u8; voice::MAX_DATAGRAM + 1];
|
||||||
|
let mut rejected: u64 = 0;
|
||||||
while let Ok((length, source)) = worker_udp.recv_from(&mut bytes).await {
|
while let Ok((length, source)) = worker_udp.recv_from(&mut bytes).await {
|
||||||
let event = {
|
let event = {
|
||||||
let Ok(mut state) = worker_state.lock() else {
|
let Ok(mut state) = worker_state.lock() else {
|
||||||
|
|
@ -40,12 +41,29 @@ impl State {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
if source != state.endpoint {
|
if source != state.endpoint {
|
||||||
|
// A relay answering from another address looks exactly like silence.
|
||||||
|
rejected += 1;
|
||||||
|
if rejected == 1 || rejected % 500 == 0 {
|
||||||
|
tracing::warn!(
|
||||||
|
%source,
|
||||||
|
expected = %state.endpoint,
|
||||||
|
rejected,
|
||||||
|
"voice datagram from an unexpected address ignored"
|
||||||
|
);
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let Ok((sequence, frame)) = voice::decode(&state.credentials, &bytes[..length])
|
let (sequence, frame) =
|
||||||
else {
|
match voice::decode(&state.credentials, &bytes[..length]) {
|
||||||
continue;
|
Ok(decoded) => decoded,
|
||||||
};
|
Err(error) => {
|
||||||
|
rejected += 1;
|
||||||
|
if rejected == 1 || rejected % 500 == 0 {
|
||||||
|
tracing::warn!(%error, rejected, "voice datagram rejected");
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
if !state.replay.accept(sequence) || frame.opus.is_empty() {
|
if !state.replay.accept(sequence) || frame.opus.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use vnox_client::audio::{IncomingVoice, PipelineConfig};
|
use vnox_client::audio::{AudioStage, IncomingVoice, PipelineConfig};
|
||||||
use vnox_client::net::{ChatMsg, NetCommand, NetEvent, NetHandle, PresenceInfo};
|
use vnox_client::net::{ChatMsg, NetCommand, NetEvent, NetHandle, PresenceInfo};
|
||||||
|
|
||||||
use crate::ui::state::{ChatMessage, ConnState, UiState, VoiceUiState};
|
use crate::ui::state::{ChatMessage, ConnState, UiState, VoiceUiState};
|
||||||
|
|
@ -304,11 +304,13 @@ pub fn process_net_events(state: &mut AppState) {
|
||||||
// Capture/playback runs independently from incoming control packets.
|
// Capture/playback runs independently from incoming control packets.
|
||||||
sync_voice_audio(state);
|
sync_voice_audio(state);
|
||||||
flush_encoded_voice(state);
|
flush_encoded_voice(state);
|
||||||
|
report_audio_issues(state);
|
||||||
|
log_audio_stats(state);
|
||||||
if let Some(pipeline) = state.audio.as_ref() {
|
if let Some(pipeline) = state.audio.as_ref() {
|
||||||
pipeline.sync_controls(vnox_client::audio::AudioSync {
|
pipeline.sync_controls(vnox_client::audio::AudioSync {
|
||||||
mic_enabled: state.ui.mic_enabled && !state.ui.deafen,
|
mic_enabled: state.ui.mic_enabled && !state.ui.deafen,
|
||||||
vad_mode: state.ui.vad_mode,
|
vad_mode: state.ui.vad_mode,
|
||||||
vad_threshold: 40.0,
|
vad_threshold: 15.0,
|
||||||
input_volume: 100.0,
|
input_volume: 100.0,
|
||||||
ptt_held: state.ui.ptt_held,
|
ptt_held: state.ui.ptt_held,
|
||||||
output_volume: if state.ui.deafen {
|
output_volume: if state.ui.deafen {
|
||||||
|
|
@ -950,3 +952,66 @@ pub fn flush_encoded_voice(state: &mut AppState) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Surfaces capture and playback failures instead of leaving a "connected" room that is mute.
|
||||||
|
fn report_audio_issues(state: &mut AppState) {
|
||||||
|
let Some(pipeline) = state.audio.as_ref() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let issues = pipeline.take_issues();
|
||||||
|
let dead = pipeline.is_dead();
|
||||||
|
for issue in issues {
|
||||||
|
let message = match issue.stage {
|
||||||
|
AudioStage::Capture => {
|
||||||
|
crate::ui::i18n::Message::with("Microphone error: {e}", vec![("e", issue.message)])
|
||||||
|
}
|
||||||
|
AudioStage::Playback => crate::ui::i18n::Message::with(
|
||||||
|
"Audio output error: {e}",
|
||||||
|
vec![("e", issue.message)],
|
||||||
|
),
|
||||||
|
};
|
||||||
|
state.ui.status_msg = Some(message);
|
||||||
|
}
|
||||||
|
if dead {
|
||||||
|
let reason = state
|
||||||
|
.ui
|
||||||
|
.status_msg
|
||||||
|
.as_ref()
|
||||||
|
.map(ToString::to_string)
|
||||||
|
.unwrap_or_default();
|
||||||
|
state.audio = None;
|
||||||
|
state.voice_channel = None;
|
||||||
|
state.ui.voice = VoiceUiState::Error(reason);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Logs signal counters every ten seconds while a voice room is active, so a bug report can
|
||||||
|
/// tell whether the signal stopped at the microphone, the network or the speaker.
|
||||||
|
fn log_audio_stats(state: &AppState) {
|
||||||
|
thread_local! {
|
||||||
|
static LAST: std::cell::Cell<Option<std::time::Instant>> = const { std::cell::Cell::new(None) };
|
||||||
|
}
|
||||||
|
let due = LAST.with(|last| {
|
||||||
|
let due = last
|
||||||
|
.get()
|
||||||
|
.is_none_or(|t| t.elapsed() >= std::time::Duration::from_secs(10));
|
||||||
|
if due {
|
||||||
|
last.set(Some(std::time::Instant::now()));
|
||||||
|
}
|
||||||
|
due
|
||||||
|
});
|
||||||
|
if !due {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(pipeline) = state.audio.as_ref() {
|
||||||
|
let stats = pipeline.stats();
|
||||||
|
tracing::info!(
|
||||||
|
captured = stats.captured_frames,
|
||||||
|
sent = stats.transmitted_frames,
|
||||||
|
received = stats.received_packets,
|
||||||
|
decoded = stats.decoded_frames,
|
||||||
|
input_peak = stats.input_peak,
|
||||||
|
"voice audio stats"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -216,6 +216,7 @@
|
||||||
"Reconnecting in {delay_secs}s (attempt {attempt})": "Повторное подключение через {delay_secs} с (попытка {attempt})",
|
"Reconnecting in {delay_secs}s (attempt {attempt})": "Повторное подключение через {delay_secs} с (попытка {attempt})",
|
||||||
"Server error: {message}": "Ошибка сервера: {message}",
|
"Server error: {message}": "Ошибка сервера: {message}",
|
||||||
"Microphone error: {e}": "Ошибка микрофона: {e}",
|
"Microphone error: {e}": "Ошибка микрофона: {e}",
|
||||||
|
"Audio output error: {e}": "Ошибка вывода звука: {e}",
|
||||||
"Cannot save audio settings: {error}": "Не удалось сохранить настройки звука: {error}",
|
"Cannot save audio settings: {error}": "Не удалось сохранить настройки звука: {error}",
|
||||||
"Passphrases do not match": "Пароли не совпадают",
|
"Passphrases do not match": "Пароли не совпадают",
|
||||||
"Passphrase must be at least 8 characters": "Пароль должен содержать минимум 8 символов",
|
"Passphrase must be at least 8 characters": "Пароль должен содержать минимум 8 символов",
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue