From 3b95df9e9c64bf028f732ce4fee38c66f6a95984 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Sat, 24 Oct 2026 23:59:27 +0200 Subject: [PATCH 1/6] fix(client): list only real audio devices and persist the choice by id The device pickers showed every ALSA PCM plugin (Discard all samples, Rate Converter, JACK, Open Sound System, surround/dmix aliases). Enumeration now goes through a pure filter that keeps the system default, the PipeWire and PulseAudio servers and one entry per hardware endpoint, with technical tails removed and a stable sort. Selections are stored by device id instead of list position, so the filter and hotplug cannot move a saved choice. Indexes saved by older versions are migrated by device name; a missing device falls back to the default. The same preferences change records whether the voice mode was chosen by the person: a profile that only carries the old push-to-talk default now starts in voice activation, explicit choices are kept. --- src/audio/devices/filter.rs | 419 +++++++++++++++++++++ src/audio/devices/input.rs | 50 +-- src/audio/devices/mod.rs | 207 +++++----- src/audio/devices/output.rs | 50 +-- src/audio/mod.rs | 12 +- src/audio/pipeline/mod.rs | 15 +- src/main.rs | 40 +- src/ui/devices.rs | 69 ++++ src/ui/mod.rs | 1 + src/ui/preferences.rs | 99 ++++- src/ui/slint/main.slint | 2 +- src/ui/state/default.rs | 6 +- src/ui/state/mod.rs | 3 + src/ui/state/types.rs | 10 + src/ui/update.rs | 4 +- translations/ru.json | 1 + translations/ru/LC_MESSAGES/vnox-client.po | 2 +- 17 files changed, 780 insertions(+), 210 deletions(-) create mode 100644 src/audio/devices/filter.rs create mode 100644 src/ui/devices.rs diff --git a/src/audio/devices/filter.rs b/src/audio/devices/filter.rs new file mode 100644 index 0000000..bd63545 --- /dev/null +++ b/src/audio/devices/filter.rs @@ -0,0 +1,419 @@ +//! Pure device-list policy: which enumerated audio devices a person should see. +//! +//! ALSA enumerates dozens of PCM "devices" that are plugins or routing aliases +//! (`null`, `lavrate`, `jack`, `oss`, `surround51:...`, `dmix:...`). Offering them in a +//! picker is noise at best and a silent-microphone trap at worst. This module keeps the +//! system default, the sound servers (PipeWire, PulseAudio) and one entry per real +//! hardware endpoint, with readable names. It does not touch any audio API, so it is +//! fully unit-testable. + +use std::cmp::Ordering; + +/// A device as reported by the audio backend, before any filtering. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawDevice { + /// Stable backend identifier (ALSA PCM name such as `sysdefault:CARD=PCH`). + pub id: String, + /// Human readable first line of the backend description. + pub name: String, +} + +/// A device that survived filtering and is offered to the user. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeviceEntry { + /// Stable backend identifier; this is what gets persisted. + pub id: String, + /// Display name shown in the picker. + pub label: String, +} + +/// How an ALSA PCM identifier should be treated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Class { + /// `pipewire` or `pulse` sound server plugin. + Server(u8), + /// A real endpoint; the number is its preference among aliases of the same endpoint. + Hardware(u8), + /// Plugins, converters, routing aliases and the default alias itself. + Hidden, +} + +const PRIORITY_HDMI: u8 = 3; +const PRIORITY_SYSDEFAULT: u8 = 3; +const PRIORITY_PLUGHW: u8 = 2; +const PRIORITY_HW: u8 = 1; + +/// Splits an ALSA PCM id into its prefix and the `CARD=..,DEV=..` selector, if any. +fn split_id(id: &str) -> (&str, Option<(&str, u32)>) { + let Some((prefix, rest)) = id.split_once(':') else { + return (id, None); + }; + let mut card = None; + let mut dev = 0u32; + for part in rest.split(',') { + if let Some(value) = part.strip_prefix("CARD=") { + card = Some(value); + } else if let Some(value) = part.strip_prefix("DEV=") { + dev = value.parse().unwrap_or(0); + } + } + (prefix, card.map(|card| (card, dev))) +} + +fn classify_alsa(id: &str) -> Class { + let (prefix, selector) = split_id(id); + match (prefix, selector) { + ("pipewire", None) => Class::Server(0), + ("pulse", None) => Class::Server(1), + ("hdmi", Some(_)) => Class::Hardware(PRIORITY_HDMI), + ("sysdefault", Some(_)) => Class::Hardware(PRIORITY_SYSDEFAULT), + ("plughw", Some(_)) => Class::Hardware(PRIORITY_PLUGHW), + ("hw", Some(_)) => Class::Hardware(PRIORITY_HW), + _ => Class::Hidden, + } +} + +/// Descriptions of known plugin pseudo-devices that must never be listed, whatever their id. +fn looks_like_plugin(name: &str) -> bool { + const MARKERS: [&str; 10] = [ + "discard all samples", + "rate converter", + "plugin", + "jack audio connection kit", + "open sound system", + "direct sample mixing", + "direct sample snooping", + "hardware device with all software conversions", + "sound server", + "default alsa", + ]; + let lowered = name.to_lowercase(); + MARKERS.iter().any(|marker| lowered.contains(marker)) +} + +/// Removes technical tails and collapses whitespace in a backend device name. +pub fn pretty_name(raw: &str) -> String { + let mut name = raw.split_whitespace().collect::>().join(" "); + for tail in [" (ALSA)", " (alsa)", " (Default Audio Device)"] { + if let Some(stripped) = name.strip_suffix(tail) { + name = stripped.trim_end().to_string(); + } + } + name +} + +fn server_label(id: &str) -> &'static str { + if id == "pipewire" { + "PipeWire" + } else { + "PulseAudio" + } +} + +fn compare_labels(a: &str, b: &str) -> Ordering { + a.to_lowercase() + .cmp(&b.to_lowercase()) + .then_with(|| a.cmp(b)) +} + +/// Filters `raw` down to the devices worth showing and returns them sorted: sound servers +/// first, then hardware alphabetically. Several aliases of one endpoint collapse into the +/// most convenient one, and equal display names never appear twice. +/// +/// `alsa` selects the ALSA policy; other backends only drop empty names, because their +/// device lists are already limited to real endpoints. +pub fn visible_devices(raw: &[RawDevice], alsa: bool) -> Vec { + struct Candidate { + entry: DeviceEntry, + server: Option, + priority: u8, + endpoint: Option<(String, u32)>, + hdmi: bool, + } + + let mut candidates: Vec = Vec::new(); + for device in raw { + let name = pretty_name(&device.name); + if device.id.is_empty() && name.is_empty() { + continue; + } + let class = if alsa { + classify_alsa(&device.id) + } else { + Class::Hardware(PRIORITY_SYSDEFAULT) + }; + let (server, priority) = match class { + Class::Hidden => continue, + Class::Server(order) => (Some(order), PRIORITY_SYSDEFAULT), + Class::Hardware(priority) => (None, priority), + }; + if server.is_none() && (name.is_empty() || (alsa && looks_like_plugin(&name))) { + continue; + } + let (prefix, selector) = split_id(&device.id); + let label = match server { + Some(_) => server_label(&device.id).to_string(), + None => name, + }; + candidates.push(Candidate { + entry: DeviceEntry { + id: device.id.clone(), + label, + }, + server, + priority, + endpoint: selector.map(|(card, dev)| (card.to_string(), dev)), + hdmi: prefix == "hdmi", + }); + } + + // One entry per physical endpoint: the best alias wins. `hdmi:` endpoints are numbered in + // their own space, so they never collapse into `hw:` aliases of the same card. + let mut chosen: Vec = Vec::new(); + for candidate in candidates { + let same = |other: &Candidate| { + candidate.server.is_some() && other.server == candidate.server + || candidate.endpoint.is_some() + && other.endpoint == candidate.endpoint + && other.hdmi == candidate.hdmi + }; + match chosen.iter().position(same) { + Some(index) if chosen[index].priority >= candidate.priority => {} + Some(index) => chosen[index] = candidate, + None => chosen.push(candidate), + } + } + + // Different ids may still read the same to a person (a card's HDMI jack listed as both + // `hdmi:` and `plughw:`); keep the preferred one. + let mut unique: Vec = Vec::new(); + for candidate in chosen { + match unique + .iter() + .position(|other| other.entry.label == candidate.entry.label) + { + Some(index) if unique[index].priority >= candidate.priority => {} + Some(index) => unique[index] = candidate, + None => unique.push(candidate), + } + } + + unique.sort_by(|a, b| match (a.server, b.server) { + (Some(x), Some(y)) => x.cmp(&y), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => compare_labels(&a.entry.label, &b.entry.label), + }); + unique.into_iter().map(|c| c.entry).collect() +} + +/// Index into the UI list (0 is the system default) for a persisted device id. +/// Unknown or empty ids select the default, so a removed device never breaks audio. +pub fn index_for_id(entries: &[DeviceEntry], saved_id: &str) -> usize { + if saved_id.is_empty() { + return 0; + } + entries + .iter() + .position(|entry| entry.id == saved_id) + .map_or(0, |position| position + 1) +} + +/// Persisted id for a UI list index (0 is the system default and persists as empty). +pub fn id_for_index(entries: &[DeviceEntry], index: usize) -> String { + index + .checked_sub(1) + .and_then(|position| entries.get(position)) + .map(|entry| entry.id.clone()) + .unwrap_or_default() +} + +/// Maps a selection saved by older versions (an index into the unfiltered, name-deduplicated +/// list with the default first) to a device id of the new list, matching by device name. +pub fn migrate_legacy_index( + legacy_names: &[String], + legacy_index: usize, + entries: &[DeviceEntry], +) -> String { + let Some(name) = legacy_index + .checked_sub(1) + .and_then(|position| legacy_names.get(position)) + else { + return String::new(); + }; + let wanted = pretty_name(name); + entries + .iter() + .find(|entry| entry.label == wanted) + .map(|entry| entry.id.clone()) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw(id: &str, name: &str) -> RawDevice { + RawDevice { + id: id.into(), + name: name.into(), + } + } + + fn typical_alsa() -> Vec { + vec![ + raw( + "null", + "Discard all samples (playback) or generate zero samples (capture)", + ), + raw( + "lavrate", + "Rate Converter Plugin Using Libav/FFmpeg Library", + ), + raw( + "samplerate", + "Rate Converter Plugin Using Samplerate Library", + ), + raw("speexrate", "Rate Converter Plugin Using Speex Resampler"), + raw("jack", "JACK Audio Connection Kit"), + raw("oss", "Open Sound System"), + raw("pipewire", "PipeWire Sound Server"), + raw("pulse", "PulseAudio Sound Server"), + raw( + "default", + "Default ALSA Output (currently PipeWire Media Server)", + ), + raw("sysdefault", "Default Audio Device"), + raw("sysdefault:CARD=PCH", "HDA Intel PCH, ALC3254 Analog"), + raw("front:CARD=PCH,DEV=0", "HDA Intel PCH, ALC3254 Analog"), + raw("surround51:CARD=PCH,DEV=0", "HDA Intel PCH, ALC3254 Analog"), + raw("hdmi:CARD=NVidia,DEV=0", "HDA NVidia, HDMI 0"), + raw("dmix:CARD=PCH,DEV=0", "HDA Intel PCH, ALC3254 Analog"), + raw("usbstream:CARD=PCH", "HDA Intel PCH"), + raw("hw:CARD=PCH,DEV=0", "HDA Intel PCH, ALC3254 Analog"), + raw("plughw:CARD=PCH,DEV=0", "HDA Intel PCH, ALC3254 Analog"), + raw("hw:CARD=NVidia,DEV=3", "HDA NVidia, HDMI 0"), + raw("plughw:CARD=NVidia,DEV=3", "HDA NVidia, HDMI 0"), + ] + } + + fn labels(entries: &[DeviceEntry]) -> Vec<&str> { + entries.iter().map(|e| e.label.as_str()).collect() + } + + #[test] + fn pseudo_devices_are_removed_and_hardware_and_servers_stay() { + let entries = visible_devices(&typical_alsa(), true); + assert_eq!( + labels(&entries), + vec![ + "PipeWire", + "PulseAudio", + "HDA Intel PCH, ALC3254 Analog", + "HDA NVidia, HDMI 0" + ] + ); + for junk in ["Discard", "Rate Converter", "JACK", "Open Sound"] { + assert!(entries.iter().all(|e| !e.label.contains(junk))); + } + } + + #[test] + fn best_alias_of_each_endpoint_wins() { + let entries = visible_devices(&typical_alsa(), true); + let ids: Vec<_> = entries.iter().map(|e| e.id.as_str()).collect(); + assert!(ids.contains(&"sysdefault:CARD=PCH")); + assert!(ids.contains(&"hdmi:CARD=NVidia,DEV=0")); + assert!(!ids.iter().any(|id| id.starts_with("hw:"))); + } + + #[test] + fn plughw_is_preferred_over_raw_hw_when_nothing_better_exists() { + let entries = visible_devices( + &[ + raw("hw:CARD=USB,DEV=0", "Blue Yeti, USB Audio"), + raw("plughw:CARD=USB,DEV=0", "Blue Yeti, USB Audio"), + ], + true, + ); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].id, "plughw:CARD=USB,DEV=0"); + } + + #[test] + fn plugin_descriptions_are_hidden_even_under_card_like_ids() { + let entries = visible_devices( + &[raw( + "hw:CARD=Weird,DEV=0", + "Rate Converter Plugin Using Foo", + )], + true, + ); + assert!(entries.is_empty()); + } + + #[test] + fn names_lose_technical_tails_and_extra_whitespace() { + assert_eq!(pretty_name(" Speakers (ALSA) "), "Speakers"); + assert_eq!(pretty_name("Headset Mic"), "Headset Mic"); + } + + #[test] + fn other_backends_keep_real_devices_and_drop_empty_names() { + let entries = visible_devices( + &[ + raw("{0.0.0}.{abc}", "Speakers (Realtek)"), + raw("{0.0.0}.{def}", ""), + raw("{0.0.0}.{ghi}", "Headphones"), + ], + false, + ); + assert_eq!(labels(&entries), vec!["Headphones", "Speakers (Realtek)"]); + } + + #[test] + fn list_is_sorted_case_insensitively_after_servers() { + let entries = visible_devices( + &[ + raw("sysdefault:CARD=b", "beta card"), + raw("sysdefault:CARD=a", "Alpha card"), + raw("pulse", "PulseAudio Sound Server"), + ], + true, + ); + assert_eq!( + labels(&entries), + vec!["PulseAudio", "Alpha card", "beta card"] + ); + } + + #[test] + fn saved_ids_resolve_to_indexes_and_back() { + let entries = visible_devices(&typical_alsa(), true); + let id = "sysdefault:CARD=PCH"; + let index = index_for_id(&entries, id); + assert_eq!(index, 3); + assert_eq!(id_for_index(&entries, index), id); + assert_eq!(index_for_id(&entries, "hw:CARD=Gone,DEV=0"), 0); + assert_eq!(index_for_id(&entries, ""), 0); + assert_eq!(id_for_index(&entries, 0), ""); + assert_eq!(id_for_index(&entries, 99), ""); + } + + #[test] + fn legacy_index_selection_migrates_by_name() { + let entries = visible_devices(&typical_alsa(), true); + let legacy = vec![ + "Discard all samples (playback) or generate zero samples (capture)".to_string(), + "HDA Intel PCH, ALC3254 Analog".to_string(), + ]; + assert_eq!( + migrate_legacy_index(&legacy, 2, &entries), + "sysdefault:CARD=PCH" + ); + // A legacy choice of a removed pseudo-device falls back to the default. + assert_eq!(migrate_legacy_index(&legacy, 1, &entries), ""); + assert_eq!(migrate_legacy_index(&legacy, 0, &entries), ""); + assert_eq!(migrate_legacy_index(&legacy, 9, &entries), ""); + } +} diff --git a/src/audio/devices/input.rs b/src/audio/devices/input.rs index 44c451a..78ad305 100644 --- a/src/audio/devices/input.rs +++ b/src/audio/devices/input.rs @@ -1,42 +1,22 @@ use anyhow::{Result, anyhow}; use cpal::Device; -use cpal::traits::{DeviceTrait, HostTrait}; +use cpal::traits::HostTrait; +use tracing::warn; -pub fn input_at(index: usize) -> Result { +/// Opens the microphone with persisted id `id`; an empty id is the system default. A saved +/// device that has disappeared falls back to the default instead of failing the join. +pub fn open_input(id: &str) -> Result { let host = cpal::default_host(); - if index == 0 { - return host - .default_input_device() - .ok_or_else(|| anyhow!("no default input device")); - } - - let devices: Vec = host.input_devices()?.collect(); - - let mut names: Vec = Vec::new(); - for dev in &devices { - if let Ok(name) = dev.description().map(|d| d.name().to_string()) - && !name.is_empty() - && !names.contains(&name) - { - names.push(name); + if !id.is_empty() { + let alsa = super::is_alsa(&host); + if let Some(device) = super::find_by_id(host.input_devices()?, id, alsa) { + return Ok(device); } + warn!( + device = id, + "saved input device not found, using the system default" + ); } - - let name = names - .get(index.saturating_sub(1)) - .ok_or_else(|| anyhow!("input device index {index} out of range"))?; - - for dev in devices { - if dev - .description() - .map(|d| d.name().to_string()) - .ok() - .as_deref() - == Some(name.as_str()) - { - return Ok(dev); - } - } - - Err(anyhow!("input device not found: {name}")) + host.default_input_device() + .ok_or_else(|| anyhow!("no default input device")) } diff --git a/src/audio/devices/mod.rs b/src/audio/devices/mod.rs index 23a19df..92ca71f 100644 --- a/src/audio/devices/mod.rs +++ b/src/audio/devices/mod.rs @@ -1,120 +1,143 @@ +//! Audio device enumeration and opening. +//! +//! The picker shows "System default" followed by the filtered endpoints from +//! [`filter::visible_devices`]. Selections are persisted by stable device id, never by list +//! position, so changing the filter or plugging hardware does not move a saved choice. + +pub mod filter; mod input; mod output; -pub use input::input_at; -pub use output::output_at; +pub use filter::{DeviceEntry, RawDevice}; +pub use input::open_input; +pub use output::open_output; use anyhow::Result; use cpal::traits::{DeviceTrait, HostTrait}; -const DEFAULT_LABEL: &str = "Default"; +/// Label of the first picker entry, translated by the UI layer. +pub const DEFAULT_LABEL: &str = "System default"; -#[derive(Debug, Clone)] -pub struct DeviceLabel { - pub display: String, - pub raw_name: String, - pub host_api: String, - pub channels: u16, -} - -#[derive(Debug, Default)] +/// Devices offered to the user, filtered and sorted. The system default is implicit and is +/// always position 0 in the UI list, so UI index `n` refers to `entries[n - 1]`. +#[derive(Debug, Default, Clone)] pub struct DeviceLists { - pub input_labels: Vec, - pub output_labels: Vec, - pub input_devices: Vec, - pub output_devices: Vec, + pub input: Vec, + pub output: Vec, } -pub fn enumerate() -> Result { - let host = cpal::default_host(); - let host_api = host.id().name().to_string(); - let (input_labels, input_devices) = list_labels(host.input_devices()?, &host_api); - let (output_labels, output_devices) = list_labels(host.output_devices()?, &host_api); - Ok(DeviceLists { - input_labels, - output_labels, - input_devices, - output_devices, +impl DeviceLists { + /// Picker labels with the system default first. + pub fn input_labels(&self) -> Vec { + labels(&self.input) + } + + /// Picker labels with the system default first. + pub fn output_labels(&self) -> Vec { + labels(&self.output) + } +} + +fn labels(entries: &[DeviceEntry]) -> Vec { + std::iter::once(DEFAULT_LABEL.to_string()) + .chain(entries.iter().map(|entry| entry.label.clone())) + .collect() +} + +fn is_alsa(host: &cpal::Host) -> bool { + host.id().name().eq_ignore_ascii_case("alsa") +} + +/// Raw identifier and display name for one device, or `None` when it cannot be described. +pub(crate) fn raw_device(device: &cpal::Device, alsa: bool) -> Option { + let description = device.description().ok()?; + let id = if alsa { + description.driver().map(str::to_owned) + } else { + None + } + .or_else(|| device.id().ok().map(|id| id.to_string()))?; + Some(RawDevice { + id, + name: description.name().to_string(), }) } -fn list_labels( - devices: impl Iterator, - host_api: &str, -) -> (Vec, Vec) { - let mut labels = vec![DEFAULT_LABEL.into()]; - let mut metadatas = vec![DeviceLabel { - display: DEFAULT_LABEL.into(), - raw_name: String::new(), - host_api: host_api.into(), - channels: 0, - }]; - for dev in devices { - let Ok(name) = dev.description().map(|d| d.name().to_string()) else { - continue; - }; - if name.is_empty() { - continue; - } - let channels = supported_channel_count(&dev); - let display = format!("{} ({})", name, host_api); - if metadatas.iter().any(|m| m.raw_name == name) { - continue; - } - labels.push(display.clone()); - metadatas.push(DeviceLabel { - display, - raw_name: name, - host_api: host_api.into(), - channels, - }); - } - (labels, metadatas) +/// Lists the visible input and output devices of the default host. +pub fn enumerate() -> Result { + let host = cpal::default_host(); + let alsa = is_alsa(&host); + let collect = |devices: Vec| { + let raw: Vec = devices + .iter() + .filter_map(|device| raw_device(device, alsa)) + .collect(); + filter::visible_devices(&raw, alsa) + }; + Ok(DeviceLists { + input: collect(host.input_devices()?.collect()), + output: collect(host.output_devices()?.collect()), + }) } -fn supported_channel_count(dev: &cpal::Device) -> u16 { - if let Ok(cfg) = dev.default_input_config() { - return cfg.channels(); +/// Names of the devices in the order older versions listed them (default excluded, names +/// deduplicated). Only used to translate a legacy saved index to a device id. +pub fn legacy_names(input: bool) -> Vec { + let host = cpal::default_host(); + let devices: Vec = if input { + host.input_devices().map(Iterator::collect) + } else { + host.output_devices().map(Iterator::collect) } - if let Ok(cfg) = dev.default_output_config() { - return cfg.channels(); + .unwrap_or_default(); + let mut names: Vec = Vec::new(); + for device in &devices { + if let Ok(name) = device.description().map(|d| d.name().to_string()) + && !name.is_empty() + && !names.contains(&name) + { + names.push(name); + } } - if let Ok(configs) = dev.supported_input_configs() - && let Some(cfg) = configs.into_iter().next() - { - return cfg.channels(); - } - if let Ok(configs) = dev.supported_output_configs() - && let Some(cfg) = configs.into_iter().next() - { - return cfg.channels(); - } - 0 + names +} + +/// Finds the device with `id` among `devices`, or `None` when it is gone. +pub(crate) fn find_by_id( + devices: impl Iterator, + id: &str, + alsa: bool, +) -> Option { + devices + .into_iter() + .find(|device| raw_device(device, alsa).is_some_and(|raw| raw.id == id)) +} + +/// Human readable name of a device for logs and diagnostics. +pub fn device_name(device: &cpal::Device) -> String { + device + .description() + .map(|d| d.name().to_owned()) + .unwrap_or_else(|_| "unknown device".into()) } #[cfg(test)] mod tests { use super::*; + #[test] - fn default_is_first_label() { - let lists = enumerate().unwrap_or_default(); - if !lists.input_labels.is_empty() { - assert_eq!(lists.input_labels[0], "Default"); - } - if !lists.output_labels.is_empty() { - assert_eq!(lists.output_labels[0], "Default"); - } - } - #[test] - fn default_meta_is_sentinel() { - let lists = enumerate().unwrap_or_default(); + fn picker_labels_start_with_the_system_default() { + let lists = DeviceLists { + input: vec![DeviceEntry { + id: "sysdefault:CARD=PCH".into(), + label: "HDA Intel PCH".into(), + }], + output: Vec::new(), + }; assert_eq!( - lists.input_devices.first().map(|m| m.raw_name.as_str()), - Some("") - ); - assert_eq!( - lists.output_devices.first().map(|m| m.raw_name.as_str()), - Some("") + lists.input_labels(), + vec!["System default", "HDA Intel PCH"] ); + assert_eq!(lists.output_labels(), vec!["System default"]); } } diff --git a/src/audio/devices/output.rs b/src/audio/devices/output.rs index 68b37eb..141b3e6 100644 --- a/src/audio/devices/output.rs +++ b/src/audio/devices/output.rs @@ -1,42 +1,22 @@ use anyhow::{Result, anyhow}; use cpal::Device; -use cpal::traits::{DeviceTrait, HostTrait}; +use cpal::traits::HostTrait; +use tracing::warn; -pub fn output_at(index: usize) -> Result { +/// Opens the output with persisted id `id`; an empty id is the system default. A saved +/// device that has disappeared falls back to the default instead of failing the join. +pub fn open_output(id: &str) -> Result { let host = cpal::default_host(); - if index == 0 { - return host - .default_output_device() - .ok_or_else(|| anyhow!("no default output device")); - } - - let devices: Vec = host.output_devices()?.collect(); - - let mut names: Vec = Vec::new(); - for dev in &devices { - if let Ok(name) = dev.description().map(|d| d.name().to_string()) - && !name.is_empty() - && !names.contains(&name) - { - names.push(name); + if !id.is_empty() { + let alsa = super::is_alsa(&host); + if let Some(device) = super::find_by_id(host.output_devices()?, id, alsa) { + return Ok(device); } + warn!( + device = id, + "saved output device not found, using the system default" + ); } - - let name = names - .get(index.saturating_sub(1)) - .ok_or_else(|| anyhow!("output device index {index} out of range"))?; - - for dev in devices { - if dev - .description() - .map(|d| d.name().to_string()) - .ok() - .as_deref() - == Some(name.as_str()) - { - return Ok(dev); - } - } - - Err(anyhow!("output device not found: {name}")) + host.default_output_device() + .ok_or_else(|| anyhow!("no default output device")) } diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 249cced..0b9b3a8 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -3,7 +3,7 @@ pub mod devices; mod pipeline; mod processing; -pub use devices::{DeviceLabel, DeviceLists}; +pub use devices::{DeviceEntry, DeviceLists}; pub use pipeline::JitterConfig; pub use pipeline::{AudioPipeline, MediaMixer}; pub use processing::{DenoiseState, TransmitGate}; @@ -25,8 +25,10 @@ pub struct AudioSync { #[derive(Debug, Clone)] pub struct PipelineConfig { pub channel_key: u64, - pub mic_device_index: usize, - pub output_device_index: usize, + /// Persisted microphone id; empty is the system default. + pub mic_device_id: String, + /// Persisted output device id; empty is the system default. + pub output_device_id: String, pub voice_bitrate_kbps: f32, pub packet_interval: usize, } @@ -34,8 +36,8 @@ pub struct PipelineConfig { impl PartialEq for PipelineConfig { fn eq(&self, other: &Self) -> bool { self.channel_key == other.channel_key - && self.mic_device_index == other.mic_device_index - && self.output_device_index == other.output_device_index + && self.mic_device_id == other.mic_device_id + && self.output_device_id == other.output_device_id && self.packet_interval == other.packet_interval } } diff --git a/src/audio/pipeline/mod.rs b/src/audio/pipeline/mod.rs index 1bcdf57..1e9e0ea 100644 --- a/src/audio/pipeline/mod.rs +++ b/src/audio/pipeline/mod.rs @@ -3,7 +3,6 @@ mod media; pub use media::MediaMixer; mod playback; -use cpal::traits::DeviceTrait; use std::sync::{ Arc, Mutex, atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, @@ -110,18 +109,12 @@ impl AudioPipeline { } pub fn start(cfg: PipelineConfig) -> anyhow::Result { - let input = devices::input_at(cfg.mic_device_index)?; - let output = devices::output_at(cfg.output_device_index)?; + let input = devices::open_input(&cfg.mic_device_id)?; + let output = devices::open_output(&cfg.output_device_id)?; info!( "audio in: {} out: {}", - input - .description() - .map(|d| d.name().to_owned()) - .unwrap_or_default(), - output - .description() - .map(|d| d.name().to_owned()) - .unwrap_or_default() + devices::device_name(&input), + devices::device_name(&output) ); let in_cfg = config::AudioConfig::find(&input, true)?; diff --git a/src/main.rs b/src/main.rs index 3ff3c53..b2e4051 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,7 +69,7 @@ fn main() -> Result<()> { } let window = MainWindow::new()?; window.set_address_input(state.borrow().ui.connect_input.as_str().into()); - refresh_devices(&window); + refresh_devices(&window, &mut state.borrow_mut().ui); let webhook_net = state.borrow().net.clone(); state.borrow_mut().webhooks = Some(Rc::new(ui::webhooks::WebhookUi::new(Rc::new( @@ -144,22 +144,27 @@ fn load_identity() -> Result> { } } -fn refresh_devices(window: &MainWindow) { - let devices = vnox_client::audio::devices::enumerate().unwrap_or_default(); - let model = |mut items: Vec| { - if items.is_empty() { - items.push(crate::ui::i18n::text("Default")); - } +fn refresh_devices(window: &MainWindow, ui: &mut UiState) { + crate::ui::devices::refresh(ui); + let model = |items: Vec| { Rc::new(slint::VecModel::from( items .into_iter() - .map(slint::SharedString::from) + .enumerate() + .map(|(index, label)| { + // Position 0 is always the system default. + slint::SharedString::from(if index == 0 { + crate::ui::i18n::text("System default") + } else { + label + }) + }) .collect::>(), )) .into() }; - window.set_input_devices(model(devices.input_labels)); - window.set_output_devices(model(devices.output_labels)); + window.set_input_devices(model(ui.device_lists.input_labels())); + window.set_output_devices(model(ui.device_lists.output_labels())); } fn wire_signals(window: &MainWindow, state: &Rc>) { @@ -220,9 +225,10 @@ fn wire_signals(window: &MainWindow, state: &Rc>) { let state = state.clone(); let weak = window.as_weak(); window.on_open_settings(move || { - state.borrow_mut().ui.settings_open = true; + let mut s = state.borrow_mut(); + s.ui.settings_open = true; if let Some(window) = weak.upgrade() { - refresh_devices(&window); + refresh_devices(&window, &mut s.ui); } }); } @@ -242,10 +248,14 @@ fn wire_signals(window: &MainWindow, state: &Rc>) { let state = state.clone(); window.on_audio_settings_changed(move |input, output, bitrate, mode, volume| { let mut s = state.borrow_mut(); - s.ui.mic_device = input.max(0) as usize; - s.ui.output_device = output.max(0) as usize; + crate::ui::devices::select(&mut s.ui, input.max(0) as usize, output.max(0) as usize); s.ui.voice_bitrate = bitrate.clamp(8.0, 128.0); - s.ui.vad_mode = mode.clamp(0, 2) as usize; + let mode = mode.clamp(0, 2) as usize; + if mode != s.ui.vad_mode { + // Only a real change counts as the person's own choice. + s.ui.vad_mode_chosen = true; + } + s.ui.vad_mode = mode; s.ui.output_volume = volume.clamp(0.0, 100.0); if let Err(error) = crate::ui::preferences::save(&s.ui) { s.ui.status_msg = Some(ui::i18n::Message::with( diff --git a/src/ui/devices.rs b/src/ui/devices.rs new file mode 100644 index 0000000..7661d62 --- /dev/null +++ b/src/ui/devices.rs @@ -0,0 +1,69 @@ +//! Bridges the persisted device ids and the positions of the settings pickers. +use vnox_client::audio::devices::{self, filter}; + +use super::state::UiState; + +/// Recomputes the picker positions from the persisted ids and the last enumeration. +pub fn sync_indexes(ui: &mut UiState) { + ui.mic_device = filter::index_for_id(&ui.device_lists.input, &ui.mic_device_id); + ui.output_device = filter::index_for_id(&ui.device_lists.output, &ui.output_device_id); +} + +/// Enumerates devices again (hardware may have changed) and keeps the saved choices. +pub fn refresh(ui: &mut UiState) { + match devices::enumerate() { + Ok(lists) => ui.device_lists = lists, + Err(error) => tracing::warn!(%error, "cannot enumerate audio devices"), + } + sync_indexes(ui); +} + +/// Applies picker positions chosen by the person, storing the matching stable ids. +pub fn select(ui: &mut UiState, input_index: usize, output_index: usize) { + ui.mic_device_id = filter::id_for_index(&ui.device_lists.input, input_index); + ui.output_device_id = filter::id_for_index(&ui.device_lists.output, output_index); + sync_indexes(ui); +} + +#[cfg(test)] +mod tests { + use super::*; + use vnox_client::audio::devices::{DeviceEntry, DeviceLists}; + + fn entry(id: &str, label: &str) -> DeviceEntry { + DeviceEntry { + id: id.into(), + label: label.into(), + } + } + + fn state() -> UiState { + UiState { + device_lists: DeviceLists { + input: vec![entry("a", "Mic A"), entry("b", "Mic B")], + output: vec![entry("x", "Out X")], + }, + ..Default::default() + } + } + + #[test] + fn selection_is_stored_by_id_and_survives_list_changes() { + let mut ui = state(); + select(&mut ui, 2, 1); + assert_eq!((ui.mic_device_id.as_str(), ui.mic_device), ("b", 2)); + assert_eq!((ui.output_device_id.as_str(), ui.output_device), ("x", 1)); + // A new device appears before the saved one; the choice follows the id. + ui.device_lists.input.insert(0, entry("new", "Aaa")); + sync_indexes(&mut ui); + assert_eq!(ui.mic_device, 3); + } + + #[test] + fn missing_device_falls_back_to_default_position() { + let mut ui = state(); + ui.mic_device_id = "unplugged".into(); + sync_indexes(&mut ui); + assert_eq!(ui.mic_device, 0); + } +} diff --git a/src/ui/mod.rs b/src/ui/mod.rs index 40761a6..f02cbf0 100644 --- a/src/ui/mod.rs +++ b/src/ui/mod.rs @@ -1,6 +1,7 @@ pub mod appearance; pub mod avatar; pub mod bridge; +pub mod devices; pub mod i18n; pub mod media; pub mod preferences; diff --git a/src/ui/preferences.rs b/src/ui/preferences.rs index 18e73c1..372aecf 100644 --- a/src/ui/preferences.rs +++ b/src/ui/preferences.rs @@ -2,15 +2,25 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; -use super::state::UiState; +use super::state::{DEFAULT_VOICE_MODE, UiState}; +use vnox_client::audio::devices::{self, filter}; #[derive(Default, Serialize, Deserialize)] #[serde(default)] struct AudioPreferences { + /// Legacy position in the old, unfiltered device list. Read only to migrate. input_device: usize, + /// Legacy position in the old, unfiltered device list. Read only to migrate. output_device: usize, + /// Stable id of the chosen microphone; empty or missing is the system default. + input_device_id: Option, + /// Stable id of the chosen output; empty or missing is the system default. + output_device_id: Option, bitrate: Option, mode: usize, + /// True once the person picked the voice input mode. Files from older versions lack it and + /// stored `mode: 0` for everybody, so an unmarked push-to-talk is not a real choice. + mode_chosen: bool, volume: Option, } @@ -20,17 +30,42 @@ fn path() -> PathBuf { .join("vnox/audio.json") } +/// Resolves the voice mode to use: an explicit choice is kept, and so is any non-default mode +/// written by older versions (those could only come from the person), but a legacy file that +/// merely records the old push-to-talk default becomes voice activation. +fn resolve_mode(prefs: &AudioPreferences) -> (usize, bool) { + let stored = prefs.mode.min(2); + if prefs.mode_chosen || stored != 0 { + (stored, true) + } else { + (DEFAULT_VOICE_MODE, false) + } +} + pub fn load(state: &mut UiState) { - let Ok(text) = std::fs::read_to_string(path()) else { - return; - }; - let Ok(prefs) = serde_json::from_str::(&text) else { - return; - }; - state.mic_device = prefs.input_device; - state.output_device = prefs.output_device; + let prefs = std::fs::read_to_string(path()) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .unwrap_or_default(); + let lists = devices::enumerate().unwrap_or_default(); + state.mic_device_id = prefs.input_device_id.clone().unwrap_or_else(|| { + filter::migrate_legacy_index( + &devices::legacy_names(true), + prefs.input_device, + &lists.input, + ) + }); + state.output_device_id = prefs.output_device_id.clone().unwrap_or_else(|| { + filter::migrate_legacy_index( + &devices::legacy_names(false), + prefs.output_device, + &lists.output, + ) + }); + state.device_lists = lists; + super::devices::sync_indexes(state); state.voice_bitrate = prefs.bitrate.unwrap_or(64.0).clamp(8.0, 128.0); - state.vad_mode = prefs.mode.min(2); + (state.vad_mode, state.vad_mode_chosen) = resolve_mode(&prefs); state.output_volume = prefs.volume.unwrap_or(90.0).clamp(0.0, 100.0); } @@ -40,12 +75,52 @@ pub fn save(state: &UiState) -> anyhow::Result<()> { std::fs::create_dir_all(parent)?; } let preferences = AudioPreferences { - input_device: state.mic_device, - output_device: state.output_device, + input_device: 0, + output_device: 0, + input_device_id: Some(state.mic_device_id.clone()), + output_device_id: Some(state.output_device_id.clone()), bitrate: Some(state.voice_bitrate), mode: state.vad_mode, + mode_chosen: state.vad_mode_chosen, volume: Some(state.output_volume), }; std::fs::write(path, serde_json::to_vec_pretty(&preferences)?)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(text: &str) -> AudioPreferences { + serde_json::from_str(text).unwrap() + } + + #[test] + fn fresh_profile_uses_voice_activation() { + assert_eq!(resolve_mode(&parse("{}")), (1, false)); + } + + #[test] + fn legacy_push_to_talk_default_becomes_voice_activation() { + assert_eq!( + resolve_mode(&parse(r#"{"mode":0,"volume":50.0}"#)), + (1, false) + ); + } + + #[test] + fn explicit_push_to_talk_is_kept() { + assert_eq!( + resolve_mode(&parse(r#"{"mode":0,"mode_chosen":true}"#)), + (0, true) + ); + } + + #[test] + fn legacy_non_default_modes_are_kept() { + assert_eq!(resolve_mode(&parse(r#"{"mode":1}"#)), (1, true)); + assert_eq!(resolve_mode(&parse(r#"{"mode":2}"#)), (2, true)); + assert_eq!(resolve_mode(&parse(r#"{"mode":9}"#)), (2, true)); + } +} diff --git a/src/ui/slint/main.slint b/src/ui/slint/main.slint index 0cf3032..7602e09 100644 --- a/src/ui/slint/main.slint +++ b/src/ui/slint/main.slint @@ -82,7 +82,7 @@ export component MainWindow inherits Window { in-out property voice-input-device: ""; in-out property voice-output-device: ""; in-out property voice-bitrate: 64.0; - in-out property vad-mode: 0; + in-out property vad-mode: 1; in-out property volume: 100.0; in-out property language-choice: "system"; callback language-selected(string); diff --git a/src/ui/state/default.rs b/src/ui/state/default.rs index 5d52840..cb68d9f 100644 --- a/src/ui/state/default.rs +++ b/src/ui/state/default.rs @@ -33,13 +33,17 @@ impl Default for UiState { mic_enabled: true, ptt_held: false, deafen: false, - vad_mode: 0, + vad_mode: crate::ui::state::DEFAULT_VOICE_MODE, voice_bitrate: 64.0, packet_interval: 1, mic_device: 0, + mic_device_id: String::new(), output_device: 0, + output_device_id: String::new(), output_volume: 90.0, + device_lists: Default::default(), + vad_mode_chosen: false, auto_reconnect: true, diff --git a/src/ui/state/mod.rs b/src/ui/state/mod.rs index 53b8061..86f317d 100644 --- a/src/ui/state/mod.rs +++ b/src/ui/state/mod.rs @@ -8,6 +8,9 @@ pub use types::{ConnState, TrustPrompt, TrustPromptKind, UiState, VoiceUiState}; use models::bookmark_label_from_address; +/// Voice input mode of a fresh profile: 0 push to talk, 1 voice activation, 2 open microphone. +pub const DEFAULT_VOICE_MODE: usize = 1; + impl UiState { pub fn remove_bookmark(&mut self, index: usize) { if index < self.bookmarks.len() { diff --git a/src/ui/state/types.rs b/src/ui/state/types.rs index 793de86..bdef68d 100644 --- a/src/ui/state/types.rs +++ b/src/ui/state/types.rs @@ -77,10 +77,20 @@ pub struct UiState { pub vad_mode: usize, pub voice_bitrate: f32, pub packet_interval: usize, + /// Picker position of the microphone (0 is the system default). pub mic_device: usize, + /// Persisted microphone id; empty is the system default. + pub mic_device_id: String, + /// Picker position of the output device (0 is the system default). pub output_device: usize, + /// Persisted output device id; empty is the system default. + pub output_device_id: String, pub output_volume: f32, + /// Devices behind the picker positions, as of the last enumeration. + pub device_lists: vnox_client::audio::DeviceLists, + /// True once the person picked a voice input mode themselves. + pub vad_mode_chosen: bool, pub auto_reconnect: bool, diff --git a/src/ui/update.rs b/src/ui/update.rs index c0b49f2..8354e25 100644 --- a/src/ui/update.rs +++ b/src/ui/update.rs @@ -865,8 +865,8 @@ fn status_handle(ui: &mut UiState, _net: &NetHandle, event: NetEvent) { pub fn pipeline_config(channel_id: &str, state: &AppState) -> PipelineConfig { PipelineConfig { channel_key: vnox_client::net::voice::channel_key(channel_id), - mic_device_index: state.ui.mic_device, - output_device_index: state.ui.output_device, + mic_device_id: state.ui.mic_device_id.clone(), + output_device_id: state.ui.output_device_id.clone(), voice_bitrate_kbps: state.ui.voice_bitrate, packet_interval: state.ui.packet_interval, } diff --git a/translations/ru.json b/translations/ru.json index dac5a8a..5618015 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -230,6 +230,7 @@ "Idle": "Нет на месте", "Do not disturb": "Не беспокоить", "Default": "По умолчанию", + "System default": "По умолчанию (системное)", "Reply to {}": "Ответ {}", "Members — {}": "Участники — {}", "Zoom {}%": "Масштаб {}%", diff --git a/translations/ru/LC_MESSAGES/vnox-client.po b/translations/ru/LC_MESSAGES/vnox-client.po index c2dcca4..77efa4e 100644 --- a/translations/ru/LC_MESSAGES/vnox-client.po +++ b/translations/ru/LC_MESSAGES/vnox-client.po @@ -543,7 +543,7 @@ msgid "Stop watching" msgstr "Закрыть просмотр" msgid "System default" -msgstr "Как в системе" +msgstr "По умолчанию (системное)" msgid "System output" msgstr "Системный звук" From 69d108b0909ee8ca8e9530929de0845d5ce00692 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Sun, 25 Oct 2026 00:18:03 +0200 Subject: [PATCH 2/6] fix(client): surface audio failures and use adaptive voice activation Capture and playback failures were only written to the log on detached threads, so a microphone or output that could not be opened still showed a connected voice room with no sound. Threads now report into a shared diagnostics sink that the UI polls: errors appear in the status bar, a room with neither working half is marked failed, and signal counters (captured, sent, received, decoded, input peak) are logged every ten seconds. A microphone that delivers only digital silence is reported, and rejected voice datagrams are logged instead of dropped silently. Voice activation now tracks the background noise floor and opens just above it, capped by a much lower default ceiling; the old fixed threshold (about -26 dBFS) rarely opened for normal speech. Stream configuration only accepts f32 formats and says so when a device offers nothing else. Playback gains a mixing queue for short notification effects. --- src/audio/config.rs | 101 +++++++++++---- src/audio/diagnostics.rs | 201 ++++++++++++++++++++++++++++++ src/audio/mod.rs | 4 +- src/audio/pipeline/capture.rs | 79 +++++++++--- src/audio/pipeline/effects.rs | 113 +++++++++++++++++ src/audio/pipeline/mod.rs | 40 +++++- src/audio/pipeline/playback.rs | 102 ++++++++------- src/audio/processing/mod.rs | 138 +++++++++++++++++++- src/net/session/handlers/voice.rs | 26 +++- src/ui/update.rs | 69 +++++++++- translations/ru.json | 1 + 11 files changed, 777 insertions(+), 97 deletions(-) create mode 100644 src/audio/diagnostics.rs create mode 100644 src/audio/pipeline/effects.rs diff --git a/src/audio/config.rs b/src/audio/config.rs index 304dd5f..0b4524b 100644 --- a/src/audio/config.rs +++ b/src/audio/config.rs @@ -1,6 +1,6 @@ use anyhow::{Result, anyhow}; use cpal::traits::DeviceTrait; -use cpal::{SampleFormat, StreamConfig}; +use cpal::{SampleFormat, StreamConfig, SupportedStreamConfigRange}; use tracing::info; pub const OPUS_RATE: u32 = 48_000; @@ -41,24 +41,19 @@ impl AudioConfig { device.supported_output_configs()?.collect() }; - // Prefer 48 kHz mono F32 — matches Opus native rate. - for r in &ranges { - if r.channels() == 1 - && r.sample_format() == SampleFormat::F32 - && r.min_sample_rate() <= OPUS_RATE - && r.max_sample_rate() >= OPUS_RATE - { - let stream = r.with_sample_rate(OPUS_RATE).config(); - info!( - "audio {}: {} Hz mono", - if is_input { "in" } else { "out" }, - OPUS_RATE - ); - return Ok(Self { - stream, - sample_rate: OPUS_RATE, - }); - } + // Streams are built with f32 callbacks, so only f32 configurations are usable. + // Prefer 48 kHz (Opus native rate) with the fewest channels, mono first. + let direction = if is_input { "in" } else { "out" }; + if let Some(range) = best_opus_range(&ranges) { + let stream = range.with_sample_rate(OPUS_RATE).config(); + info!( + "audio {direction}: {OPUS_RATE} Hz {}ch f32", + stream.channels + ); + return Ok(Self { + stream, + sample_rate: OPUS_RATE, + }); } // Fallback: device native rate, resample to/from Opus in capture/playback. @@ -68,14 +63,20 @@ impl AudioConfig { device.default_output_config() } .map_err(|e| anyhow!("no default audio config: {e}"))?; + if default.sample_format() != SampleFormat::F32 { + return Err(anyhow!( + "the {} device offers {:?} samples but 32-bit float is required; \ + pick another device in the audio settings", + if is_input { "input" } else { "output" }, + default.sample_format() + )); + } let sample_rate = default.sample_rate(); let stream: StreamConfig = default.config(); info!( - "audio {}: {} Hz {}ch (will resample for Opus)", - if is_input { "in" } else { "out" }, - sample_rate, - stream.channels + "audio {direction}: {} Hz {}ch (will resample for Opus)", + sample_rate, stream.channels ); Ok(Self { stream, @@ -84,6 +85,18 @@ impl AudioConfig { } } +/// Picks the f32 range that covers 48 kHz with the fewest channels (mono preferred). +fn best_opus_range(ranges: &[SupportedStreamConfigRange]) -> Option<&SupportedStreamConfigRange> { + ranges + .iter() + .filter(|r| { + r.sample_format() == SampleFormat::F32 + && r.min_sample_rate() <= OPUS_RATE + && r.max_sample_rate() >= OPUS_RATE + }) + .min_by_key(|r| (r.channels() != 1, r.channels())) +} + pub fn frame_samples(sample_rate: u32) -> usize { (sample_rate as usize * FRAME_MS as usize) / 1000 } @@ -118,7 +131,47 @@ pub fn resample_rate(input: &[f32], from_hz: u32, to_hz: u32) -> Vec { #[cfg(test)] mod tests { - use super::{bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting}; + use super::{ + OPUS_RATE, SampleFormat, SupportedStreamConfigRange, best_opus_range, + bitrate_bps_from_kbps, frame_ms_from_interval, jitter_ms_from_setting, + }; + + use cpal::SupportedBufferSize; + + fn range( + channels: u16, + min: u32, + max: u32, + format: SampleFormat, + ) -> SupportedStreamConfigRange { + SupportedStreamConfigRange::new(channels, min, max, SupportedBufferSize::Unknown, format) + } + + #[test] + fn config_prefers_mono_f32_at_the_opus_rate() { + let ranges = [ + range(2, 1, 384_000, SampleFormat::F32), + range(1, 1, 384_000, SampleFormat::I16), + range(1, 1, 384_000, SampleFormat::F32), + ]; + let best = best_opus_range(&ranges).expect("f32 range"); + assert_eq!( + (best.channels(), best.sample_format()), + (1, SampleFormat::F32) + ); + } + + #[test] + fn config_falls_back_to_stereo_and_rejects_non_f32_or_wrong_rate() { + let stereo = [range(2, 8_000, 96_000, SampleFormat::F32)]; + assert_eq!(best_opus_range(&stereo).map(|r| r.channels()), Some(2)); + let unusable = [ + range(1, 8_000, 44_100, SampleFormat::F32), + range(1, 1, 96_000, SampleFormat::I16), + ]; + assert!(best_opus_range(&unusable).is_none()); + assert_eq!(OPUS_RATE, 48_000); + } #[test] fn jitter_size_maps_to_ms() { diff --git a/src/audio/diagnostics.rs b/src/audio/diagnostics.rs new file mode 100644 index 0000000..aeb0f96 --- /dev/null +++ b/src/audio/diagnostics.rs @@ -0,0 +1,201 @@ +//! Shared audio health: failures raised on capture and playback threads must reach the UI. +//! +//! Earlier versions only logged such failures (`tracing::error!` on a detached thread), so a +//! microphone that could not be opened still showed a connected voice room with no sound. The +//! threads now report into [`Diagnostics`]; the UI polls it and shows the message. + +use std::sync::{ + Mutex, + atomic::{AtomicU32, AtomicU64, Ordering}, +}; + +/// Which half of the audio pipeline raised an issue. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioStage { + Capture, + Playback, +} + +/// One reportable audio problem. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioIssue { + pub stage: AudioStage, + pub message: String, + /// The stage cannot continue (stream not built, thread ended). + pub fatal: bool, +} + +/// Counters for support logs: where in the chain did the signal stop? +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct AudioStats { + pub captured_frames: u64, + pub transmitted_frames: u64, + pub received_packets: u64, + pub decoded_frames: u64, + /// Loudest input sample seen since the last snapshot, 0.0 to 1.0. + pub input_peak: f32, +} + +#[derive(Default)] +struct Failures { + capture: bool, + playback: bool, +} + +/// Thread-safe sink for issues and counters, shared by the pipeline threads. +#[derive(Default)] +pub struct Diagnostics { + issues: Mutex>, + failed: Mutex, + last_message: Mutex>, + captured_frames: AtomicU64, + transmitted_frames: AtomicU64, + received_packets: AtomicU64, + decoded_frames: AtomicU64, + input_peak_bits: AtomicU32, +} + +impl Diagnostics { + /// Records an issue (and logs it). Identical consecutive reports from one stage are + /// collapsed so a failing device callback cannot flood the UI. + pub fn report(&self, stage: AudioStage, fatal: bool, message: impl Into) { + let message = message.into(); + if fatal && let Ok(mut failed) = self.failed.lock() { + match stage { + AudioStage::Capture => failed.capture = true, + AudioStage::Playback => failed.playback = true, + } + } + if let Ok(mut last) = self.last_message.lock() { + if last + .as_ref() + .is_some_and(|(s, m)| *s == stage && *m == message) + { + return; + } + *last = Some((stage, message.clone())); + } + tracing::error!(?stage, fatal, "audio: {message}"); + if let Ok(mut issues) = self.issues.lock() { + if issues.len() >= 16 { + issues.remove(0); + } + issues.push(AudioIssue { + stage, + message, + fatal, + }); + } + } + + /// Drains the issues reported since the last call. + pub fn take_issues(&self) -> Vec { + self.issues + .lock() + .map(|mut issues| std::mem::take(&mut *issues)) + .unwrap_or_default() + } + + /// True when neither capture nor playback is working any more. + pub fn all_failed(&self) -> bool { + self.failed + .lock() + .map(|failed| failed.capture && failed.playback) + .unwrap_or(false) + } + + pub fn count_captured(&self, peak: f32) { + self.captured_frames.fetch_add(1, Ordering::Relaxed); + let mut current = self.input_peak_bits.load(Ordering::Relaxed); + while peak > f32::from_bits(current) { + match self.input_peak_bits.compare_exchange_weak( + current, + peak.to_bits(), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + + pub fn count_transmitted(&self) { + self.transmitted_frames.fetch_add(1, Ordering::Relaxed); + } + + pub fn count_received(&self) { + self.received_packets.fetch_add(1, Ordering::Relaxed); + } + + pub fn count_decoded(&self) { + self.decoded_frames.fetch_add(1, Ordering::Relaxed); + } + + /// Current counters; the input peak is reset so each snapshot covers one interval. + pub fn snapshot(&self) -> AudioStats { + AudioStats { + captured_frames: self.captured_frames.load(Ordering::Relaxed), + transmitted_frames: self.transmitted_frames.load(Ordering::Relaxed), + received_packets: self.received_packets.load(Ordering::Relaxed), + decoded_frames: self.decoded_frames.load(Ordering::Relaxed), + input_peak: f32::from_bits(self.input_peak_bits.swap(0, Ordering::Relaxed)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn issues_are_delivered_once_and_duplicates_collapse() { + let diag = Diagnostics::default(); + diag.report(AudioStage::Capture, true, "no microphone"); + diag.report(AudioStage::Capture, true, "no microphone"); + diag.report(AudioStage::Playback, false, "xrun"); + let issues = diag.take_issues(); + assert_eq!(issues.len(), 2); + assert_eq!(issues[0].stage, AudioStage::Capture); + assert!(issues[0].fatal); + assert!(diag.take_issues().is_empty()); + } + + #[test] + fn all_failed_requires_both_stages_to_fail_fatally() { + let diag = Diagnostics::default(); + diag.report(AudioStage::Capture, true, "a"); + assert!(!diag.all_failed()); + diag.report(AudioStage::Playback, false, "b"); + assert!(!diag.all_failed()); + diag.report(AudioStage::Playback, true, "c"); + assert!(diag.all_failed()); + } + + #[test] + fn snapshot_reports_counters_and_resets_the_peak() { + let diag = Diagnostics::default(); + diag.count_captured(0.2); + diag.count_captured(0.5); + diag.count_captured(0.1); + diag.count_transmitted(); + diag.count_received(); + diag.count_decoded(); + let first = diag.snapshot(); + assert_eq!(first.captured_frames, 3); + assert_eq!(first.transmitted_frames, 1); + assert_eq!(first.received_packets, 1); + assert_eq!(first.decoded_frames, 1); + assert!((first.input_peak - 0.5).abs() < f32::EPSILON); + assert_eq!(diag.snapshot().input_peak, 0.0); + } + + #[test] + fn issue_queue_is_bounded() { + let diag = Diagnostics::default(); + for i in 0..40 { + diag.report(AudioStage::Playback, false, format!("e{i}")); + } + assert_eq!(diag.take_issues().len(), 16); + } +} diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 0b9b3a8..97b9b4e 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -1,12 +1,14 @@ pub mod config; pub mod devices; +pub mod diagnostics; mod pipeline; mod processing; pub use devices::{DeviceEntry, DeviceLists}; +pub use diagnostics::{AudioIssue, AudioStage, AudioStats}; pub use pipeline::JitterConfig; pub use pipeline::{AudioPipeline, MediaMixer}; -pub use processing::{DenoiseState, TransmitGate}; +pub use processing::{DenoiseState, TransmitGate, VoiceActivity}; #[derive(Debug, Clone, Copy)] pub struct AudioSync { diff --git a/src/audio/pipeline/capture.rs b/src/audio/pipeline/capture.rs index a64615c..45b6977 100644 --- a/src/audio/pipeline/capture.rs +++ b/src/audio/pipeline/capture.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use cpal::traits::{DeviceTrait, StreamTrait}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; @@ -7,9 +7,13 @@ use tracing::{info, warn}; use crate::audio::{ EncodedFrame, config::{self, AudioConfig, OPUS_RATE}, - processing::{DenoiseState, TransmitGate}, + diagnostics::{AudioStage, Diagnostics}, + processing::{DenoiseState, TransmitGate, VoiceActivity}, }; +/// Seconds of exact digital silence after which the microphone is reported as dead. +const SILENCE_WARNING_SECS: u32 = 4; + const MAX_OPUS: usize = 4000; #[allow(clippy::too_many_arguments)] @@ -24,6 +28,7 @@ pub fn run( local_speaking: Arc, stopped: Arc, microphone_tap: super::media::MicrophoneTap, + diagnostics: Arc, ) -> Result<()> { let mut encoder = opus::Encoder::new(OPUS_RATE, opus::Channels::Mono, opus::Application::Voip)?; let mut last_bitrate_bps: u32 = 0; @@ -41,24 +46,38 @@ pub fn run( let mut voice_seq: u32 = 0; let mut timestamp: u32 = 0; - let stream = device.build_input_stream( - cfg.stream, - move |data: &[f32], _| { - let mut b = buf2.lock().unwrap(); - if in_channels <= 1 { - b.extend_from_slice(data); - } else { - for frame in data.chunks(in_channels) { - let sum: f32 = frame.iter().sum(); - b.push(sum / in_channels as f32); + let error_diagnostics = diagnostics.clone(); + let stream = device + .build_input_stream( + cfg.stream, + move |data: &[f32], _| { + let mut b = buf2.lock().unwrap(); + if in_channels <= 1 { + b.extend_from_slice(data); + } else { + for frame in data.chunks(in_channels) { + let sum: f32 = frame.iter().sum(); + b.push(sum / in_channels as f32); + } } - } - }, - |e| tracing::error!("input: {e}"), - None, - )?; - stream.play()?; + }, + move |e| { + error_diagnostics.report( + AudioStage::Capture, + false, + format!("microphone stream: {e}"), + ) + }, + None, + ) + .context("cannot open the microphone stream")?; + stream + .play() + .context("cannot start the microphone stream")?; info!("capture started ({} Hz)", cfg.sample_rate); + let mut vad = VoiceActivity::default(); + let mut silent_frames: u32 = 0; + let mut silence_reported = false; while !stopped.load(Ordering::Acquire) { std::thread::sleep(std::time::Duration::from_millis(5)); @@ -80,12 +99,29 @@ pub fn run( padded }; + let peak = raw_peak(&pcm); + diagnostics.count_captured(peak); + if peak == 0.0 { + silent_frames = silent_frames.saturating_add(1); + if !silence_reported && silent_frames * frame_ms >= SILENCE_WARNING_SECS * 1000 { + silence_reported = true; + diagnostics.report( + AudioStage::Capture, + false, + "the microphone delivers only silence; check the input device and its mute switch", + ); + } + } else { + silent_frames = 0; + silence_reported = false; + } + let gate_snapshot = gate.lock().unwrap().clone(); TransmitGate::apply_input_volume(&mut pcm, gate_snapshot.input_volume); if gate_snapshot.noise_suppress { denoise.process(&mut pcm); } - let speaking = TransmitGate::should_transmit(&gate_snapshot, &pcm); + let speaking = gate_snapshot.decide(&pcm, &mut vad, frame_ms); local_speaking.store(speaking, Ordering::Relaxed); if !speaking { b = buf.lock().unwrap(); @@ -110,6 +146,7 @@ pub fn run( { return Ok(()); } + diagnostics.count_transmitted(); voice_seq = voice_seq.wrapping_add(1); timestamp = timestamp.wrapping_add(opus_frame as u32); } @@ -121,6 +158,10 @@ pub fn run( Ok(()) } +fn raw_peak(samples: &[f32]) -> f32 { + samples.iter().fold(0.0f32, |peak, s| peak.max(s.abs())) +} + fn frame_samples_at(sample_rate: u32, frame_ms: u32) -> usize { (sample_rate as usize * frame_ms as usize) / 1000 } diff --git a/src/audio/pipeline/effects.rs b/src/audio/pipeline/effects.rs new file mode 100644 index 0000000..403db1f --- /dev/null +++ b/src/audio/pipeline/effects.rs @@ -0,0 +1,113 @@ +//! Short notification sounds mixed into the already open playback stream. +//! +//! No extra device is opened while a voice room is active, so effects follow the selected +//! output device and cannot conflict with exclusive hardware access. The audio callback only +//! pops samples with `try_lock`; it never waits for the UI thread. + +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, +}; + +use crate::audio::config::{OPUS_RATE, resample_rate}; + +/// Longest backlog of queued effect audio (two seconds) so a burst cannot grow without bound. +const MAX_SECONDS: usize = 2; + +#[derive(Clone)] +pub struct EffectQueue { + samples: Arc>>, + device_rate: u32, +} + +impl EffectQueue { + pub(super) fn new(device_rate: u32) -> Self { + Self { + samples: Arc::default(), + device_rate, + } + } + + /// Mixes a mono 48 kHz buffer (volume already applied) on top of what is still queued. + pub fn push(&self, mono_48k: &[f32]) { + let converted = if self.device_rate == OPUS_RATE { + mono_48k.to_vec() + } else { + resample_rate(mono_48k, OPUS_RATE, self.device_rate) + }; + let limit = self.device_rate as usize * MAX_SECONDS; + let Ok(mut queue) = self.samples.lock() else { + return; + }; + for (index, sample) in converted.into_iter().take(limit).enumerate() { + if let Some(existing) = queue.get_mut(index) { + *existing += sample; + } else { + queue.push_back(sample); + } + } + } + + /// Adds queued effect samples to an interleaved output buffer. + pub(super) fn mix_into(&self, out: &mut [f32], channels: usize) { + let Ok(mut queue) = self.samples.try_lock() else { + return; + }; + if queue.is_empty() || channels == 0 { + return; + } + for frame in out.chunks_mut(channels) { + let Some(sample) = queue.pop_front() else { + break; + }; + for value in frame { + *value = (*value + sample).clamp(-1.0, 1.0); + } + } + } + + pub(super) fn clear(&self) { + if let Ok(mut queue) = self.samples.lock() { + queue.clear(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn effects_play_once_on_every_channel_and_then_stop() { + let queue = EffectQueue::new(48_000); + queue.push(&[0.5, 0.25]); + let mut out = [0.0f32; 8]; + queue.mix_into(&mut out, 2); + assert_eq!(out, [0.5, 0.5, 0.25, 0.25, 0.0, 0.0, 0.0, 0.0]); + let mut again = [0.0f32; 4]; + queue.mix_into(&mut again, 2); + assert_eq!(again, [0.0; 4]); + } + + #[test] + fn overlapping_effects_add_and_the_output_is_clamped() { + let queue = EffectQueue::new(48_000); + queue.push(&[0.4, 0.4]); + queue.push(&[0.4, 0.4, 0.4]); + let mut out = [0.9f32; 3]; + queue.mix_into(&mut out, 1); + assert_eq!(out, [1.0, 1.0, 1.0]); + } + + #[test] + fn queue_is_bounded_and_resampled_to_the_device_rate() { + let queue = EffectQueue::new(24_000); + queue.push(&vec![0.1; 48_000]); + let length = queue.samples.lock().unwrap().len(); + assert_eq!(length, 24_000); + queue.clear(); + assert!(queue.samples.lock().unwrap().is_empty()); + queue.push(&vec![0.1; 48_000 * 10]); + assert_eq!(queue.samples.lock().unwrap().len(), 24_000 * MAX_SECONDS); + } +} diff --git a/src/audio/pipeline/mod.rs b/src/audio/pipeline/mod.rs index 1e9e0ea..1a7f4f7 100644 --- a/src/audio/pipeline/mod.rs +++ b/src/audio/pipeline/mod.rs @@ -1,4 +1,5 @@ mod capture; +mod effects; mod media; pub use media::MediaMixer; mod playback; @@ -12,6 +13,7 @@ use tracing::info; use crate::audio::{ AudioSync, EncodedFrame, IncomingVoice, PipelineConfig, config, devices, + diagnostics::{AudioIssue, AudioStage, AudioStats, Diagnostics}, processing::TransmitGate, }; @@ -22,6 +24,8 @@ pub struct AudioPipeline { encoded_rx: Receiver, microphone_tap: media::MicrophoneTap, media_mixer: MediaMixer, + effects: effects::EffectQueue, + diagnostics: Arc, voice_tx: Sender, _encoded_tx: Sender, gate: Arc>, @@ -40,6 +44,7 @@ impl Drop for AudioPipeline { self.stopped.store(true, Ordering::Release); self.microphone_tap.enable(false); self.media_mixer.clear(); + self.effects.clear(); } } @@ -54,6 +59,27 @@ impl AudioPipeline { self.media_mixer.clone() } + /// Plays a short notification (mono, 48 kHz, volume already applied) through the open + /// playback stream without touching the voice path. + pub fn play_effect(&self, mono_48k: &[f32]) { + self.effects.push(mono_48k); + } + + /// Problems raised by the capture and playback threads since the last call. + pub fn take_issues(&self) -> Vec { + self.diagnostics.take_issues() + } + + /// True when neither the microphone nor the output could be used. + pub fn is_dead(&self) -> bool { + self.diagnostics.all_failed() + } + + /// Signal counters for support logs. + pub fn stats(&self) -> AudioStats { + self.diagnostics.snapshot() + } + pub fn poll_encoded(&self) -> Option { self.encoded_rx.try_recv().ok() } @@ -124,6 +150,11 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { let tap_capture = microphone_tap.clone(); let media_mixer = MediaMixer::new(out_cfg.sample_rate); let mixer_playback = media_mixer.clone(); + let effects = effects::EffectQueue::new(out_cfg.sample_rate); + let effects_playback = effects.clone(); + let diagnostics = Arc::new(Diagnostics::default()); + let diagnostics_capture = diagnostics.clone(); + let diagnostics_playback = diagnostics.clone(); let (encoded_tx, encoded_rx) = mpsc::channel::(); let (voice_tx, voice_rx) = mpsc::channel::(); @@ -162,8 +193,9 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { local_speaking_cap, capture_stopped, tap_capture, + diagnostics_capture.clone(), ) { - tracing::error!("capture: {e}"); + diagnostics_capture.report(AudioStage::Capture, true, format!("{e:#}")); } }); std::thread::spawn(move || { @@ -175,8 +207,10 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { jitter_play, last_remote_voice_play, mixer_playback, + effects_playback, + diagnostics_playback.clone(), ) { - tracing::error!("playback: {e}"); + diagnostics_playback.report(AudioStage::Playback, true, format!("{e:#}")); } }); @@ -185,6 +219,8 @@ pub fn start(cfg: PipelineConfig) -> anyhow::Result { encoded_rx, microphone_tap, media_mixer, + effects, + diagnostics, voice_tx, _encoded_tx: encoded_tx, gate, diff --git a/src/audio/pipeline/playback.rs b/src/audio/pipeline/playback.rs index 2d9291a..817c08d 100644 --- a/src/audio/pipeline/playback.rs +++ b/src/audio/pipeline/playback.rs @@ -1,5 +1,5 @@ use crate::jitter::{BufferedPacket, JitterBuffer}; -use anyhow::Result; +use anyhow::{Context, Result}; use cpal::traits::{DeviceTrait, StreamTrait}; use std::sync::{ Arc, Mutex, @@ -11,6 +11,7 @@ use tracing::{info, warn}; use crate::audio::IncomingVoice; use crate::audio::config::{self, AudioConfig, OPUS_RATE}; +use crate::audio::diagnostics::{AudioStage, Diagnostics}; #[derive(Debug, Clone)] pub struct JitterConfig { @@ -27,6 +28,8 @@ pub fn run( jitter_cfg: Arc>, last_remote_voice_ms: Arc, media_mixer: super::media::MediaMixer, + effects: super::effects::EffectQueue, + diagnostics: Arc, ) -> Result<()> { let mut decoder = opus::Decoder::new(OPUS_RATE, opus::Channels::Mono)?; let pb: Arc>> = Arc::new(Mutex::new(Vec::new())); @@ -37,51 +40,60 @@ pub fn run( let opus_frame = config::frame_samples(OPUS_RATE); let media_queues = media_mixer.queues.clone(); - let stream = device.build_output_stream( - cfg.stream, - move |out: &mut [f32], _| { - let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0); - let mut buf = pb2.lock().unwrap(); - if out_channels == 1 { - let n = buf.len().min(out.len()); - for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) { - *o = s * gain; + let error_diagnostics = diagnostics.clone(); + let effects_playback = effects.clone(); + let stream = device + .build_output_stream( + cfg.stream, + move |out: &mut [f32], _| { + let gain = (*vol2.lock().unwrap() / 100.0).clamp(0.0, 2.0); + let mut buf = pb2.lock().unwrap(); + if out_channels == 1 { + let n = buf.len().min(out.len()); + for (o, s) in out[..n].iter_mut().zip(buf[..n].iter()) { + *o = s * gain; + } + buf.drain(..n); + out[n..].fill(0.0); + } else { + let frames = out.len() / out_channels; + for i in 0..frames { + let sample = if i < buf.len() { buf[i] * gain } else { 0.0 }; + for ch in 0..out_channels { + out[i * out_channels + ch] = sample; + } + } + let drain_n = frames.min(buf.len()); + buf.drain(..drain_n); } - buf.drain(..n); - out[n..].fill(0.0); - } else { - let frames = out.len() / out_channels; - for i in 0..frames { - let sample = if i < buf.len() { buf[i] * gain } else { 0.0 }; - for ch in 0..out_channels { - out[i * out_channels + ch] = sample; + effects_playback.mix_into(out, out_channels); + if let Ok(mut queues) = media_queues.lock() { + for frame in out.chunks_mut(out_channels) { + let mut mixed = [0.0f32; 2]; + for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) { + mixed[0] += samples[0]; + mixed[1] += samples[1]; + } + for (channel, sample) in frame.iter_mut().enumerate() { + let value = if out_channels == 1 || channel > 1 { + (mixed[0] + mixed[1]) * 0.5 + } else { + mixed[channel] + }; + *sample = (*sample + value * gain).clamp(-1.0, 1.0); + } } } - let drain_n = frames.min(buf.len()); - buf.drain(..drain_n); - } - if let Ok(mut queues) = media_queues.lock() { - for frame in out.chunks_mut(out_channels) { - let mut mixed = [0.0f32; 2]; - for samples in queues.values_mut().filter_map(|queue| queue.pop_front()) { - mixed[0] += samples[0]; - mixed[1] += samples[1]; - } - for (channel, sample) in frame.iter_mut().enumerate() { - let value = if out_channels == 1 || channel > 1 { - (mixed[0] + mixed[1]) * 0.5 - } else { - mixed[channel] - }; - *sample = (*sample + value * gain).clamp(-1.0, 1.0); - } - } - } - }, - |e| tracing::error!("output: {e}"), - None, - )?; - stream.play()?; + }, + move |e| { + error_diagnostics.report(AudioStage::Playback, false, format!("output stream: {e}")) + }, + None, + ) + .context("cannot open the audio output stream")?; + stream + .play() + .context("cannot start the audio output stream")?; info!("playback started ({} Hz)", cfg.sample_rate); let clock = Instant::now(); @@ -121,6 +133,7 @@ pub fn run( opus_data: pkt.opus_data, arrived_at: clock.elapsed().as_millis() as u64, }); + diagnostics.count_received(); } Err(TryRecvError::Empty) => break, Err(TryRecvError::Disconnected) => { @@ -138,6 +151,7 @@ pub fn run( needs_resample, cfg.sample_rate, &pb, + &diagnostics, ); } std::thread::sleep(Duration::from_millis(2)); @@ -152,6 +166,7 @@ fn decode_into_playback( needs_resample: bool, device_rate: u32, pb: &Arc>>, + diagnostics: &Diagnostics, ) { let mut pcm = vec![0f32; opus_frame]; match decoder.decode_float(data, &mut pcm, false) { @@ -163,6 +178,7 @@ fn decode_into_playback( pcm }; pb.lock().unwrap().extend_from_slice(&samples); + diagnostics.count_decoded(); } Err(e) => warn!("decode: {e}"), } diff --git a/src/audio/processing/mod.rs b/src/audio/processing/mod.rs index 4375601..9e58369 100644 --- a/src/audio/processing/mod.rs +++ b/src/audio/processing/mod.rs @@ -15,8 +15,8 @@ impl Default for TransmitGate { fn default() -> Self { Self { mic_enabled: true, - vad_mode: 0, - vad_threshold: 40.0, + vad_mode: 1, + vad_threshold: 15.0, ptt_held: false, input_volume: 80.0, noise_suppress: false, @@ -62,6 +62,59 @@ fn vad_rms_threshold(ui_threshold: f32) -> f32 { 0.002 + (ui_threshold.clamp(0.0, 100.0) / 100.0) * 0.118 } +/// Lowest RMS the automatic threshold may settle on (about -48 dBFS). +const VAD_MIN_THRESHOLD: f32 = 0.004; +/// How long transmission continues after the signal drops below the threshold. +const VAD_HANGOVER_MS: u32 = 300; + +/// Voice activity detector with an automatically tracked noise floor. +/// +/// The user threshold is only a ceiling: the detector opens a little above the measured +/// background noise, so quiet microphones work without manual tuning and noisy rooms do not +/// keep the channel open. A short hangover avoids clipping word endings. +#[derive(Debug, Clone)] +pub struct VoiceActivity { + noise_floor: f32, + hangover_ms: u32, +} + +impl Default for VoiceActivity { + fn default() -> Self { + Self { + noise_floor: 0.004, + hangover_ms: 0, + } + } +} + +impl VoiceActivity { + /// Current opening threshold for the given ceiling. + pub fn threshold(&self, ceiling: f32) -> f32 { + (self.noise_floor * 3.0 + 0.002) + .max(VAD_MIN_THRESHOLD) + .min(ceiling.max(VAD_MIN_THRESHOLD)) + } + + /// Feeds one frame (`frame_ms` long) and tells whether it should be transmitted. + pub fn update(&mut self, rms: f32, ceiling: f32, frame_ms: u32) -> bool { + let threshold = self.threshold(ceiling); + if rms >= threshold { + self.hangover_ms = VAD_HANGOVER_MS; + // Sustained sound the detector cannot tell from speech slowly raises the floor, so a + // fan that starts mid-call stops holding the channel open. + self.noise_floor += (rms.min(ceiling / 3.0) - self.noise_floor).max(0.0) * 0.0005; + return true; + } + let rate = if rms < self.noise_floor { 0.2 } else { 0.02 }; + self.noise_floor += (rms - self.noise_floor) * rate; + if self.hangover_ms > 0 { + self.hangover_ms = self.hangover_ms.saturating_sub(frame_ms); + return true; + } + false + } +} + impl TransmitGate { pub fn apply_input_volume(pcm: &mut [f32], volume_pct: f32) { let gain = (volume_pct / 100.0).clamp(0.0, 2.0); @@ -73,6 +126,21 @@ impl TransmitGate { } } + /// Like [`should_transmit`](Self::should_transmit) but voice activation uses the adaptive + /// detector in `vad` instead of a fixed threshold. + pub fn decide(&self, pcm: &[f32], vad: &mut VoiceActivity, frame_ms: u32) -> bool { + if self.mic_enabled && self.vad_mode == 1 { + // The detector must see every frame to track the noise floor, even when muted + // frames are skipped above. + return vad.update( + frame_rms(pcm), + vad_rms_threshold(self.vad_threshold), + frame_ms, + ); + } + self.should_transmit(pcm) + } + pub fn should_transmit(&self, pcm: &[f32]) -> bool { if !self.mic_enabled { return false; @@ -129,6 +197,72 @@ mod tests { assert!(gate.should_transmit(&loud_frame())); } + #[test] + fn detector_adapts_to_a_quiet_microphone() { + let mut vad = VoiceActivity::default(); + // Room tone at 0.0005 RMS for one second lets the floor settle. + for _ in 0..50 { + vad.update(0.0005, 0.02, 20); + } + assert!(vad.threshold(0.02) <= VAD_MIN_THRESHOLD + 1e-6); + // Quiet speech at 0.008 RMS would never pass a fixed 0.05 threshold. + assert!(vad.update(0.008, 0.02, 20)); + } + + #[test] + fn detector_never_exceeds_the_user_ceiling() { + let mut vad = VoiceActivity::default(); + for _ in 0..2000 { + vad.update(0.5, 0.02, 20); + } + assert!(vad.threshold(0.02) <= 0.02 + 1e-6); + } + + #[test] + fn detector_keeps_transmitting_through_the_hangover_then_stops() { + let mut vad = VoiceActivity::default(); + for _ in 0..20 { + vad.update(0.0003, 0.02, 20); + } + assert!(vad.update(0.05, 0.02, 20)); + let mut open_frames = 0; + while vad.update(0.0003, 0.02, 20) { + open_frames += 1; + assert!(open_frames < 100); + } + assert_eq!(open_frames, (VAD_HANGOVER_MS / 20) as usize); + } + + #[test] + fn default_gate_uses_voice_activation_with_a_sane_ceiling() { + let gate = TransmitGate::default(); + assert_eq!(gate.vad_mode, 1); + // The ceiling must be low enough for ordinary speech levels. + assert!(vad_rms_threshold(gate.vad_threshold) < 0.03); + } + + #[test] + fn decide_gates_voice_activation_by_mic_state_and_mode() { + let mut vad = VoiceActivity::default(); + let loud = loud_frame(); + let gate = TransmitGate { + vad_mode: 1, + ..Default::default() + }; + assert!(gate.decide(&loud, &mut vad, 20)); + let muted = TransmitGate { + mic_enabled: false, + vad_mode: 1, + ..Default::default() + }; + assert!(!muted.decide(&loud, &mut vad, 20)); + let ptt = TransmitGate { + vad_mode: 0, + ..Default::default() + }; + assert!(!ptt.decide(&loud, &mut vad, 20)); + } + #[test] fn vad_uses_threshold() { let gate = TransmitGate { diff --git a/src/net/session/handlers/voice.rs b/src/net/session/handlers/voice.rs index 3da58ff..9be37cc 100644 --- a/src/net/session/handlers/voice.rs +++ b/src/net/session/handlers/voice.rs @@ -31,6 +31,7 @@ impl State { let events = event_tx.clone(); let receiver = tokio::spawn(async move { let mut bytes = vec![0u8; voice::MAX_DATAGRAM + 1]; + let mut rejected: u64 = 0; while let Ok((length, source)) = worker_udp.recv_from(&mut bytes).await { let event = { let Ok(mut state) = worker_state.lock() else { @@ -40,12 +41,29 @@ impl State { continue; }; if source != state.endpoint { + // A relay answering from another address looks exactly like silence. + rejected += 1; + if rejected == 1 || rejected % 500 == 0 { + tracing::warn!( + %source, + expected = %state.endpoint, + rejected, + "voice datagram from an unexpected address ignored" + ); + } continue; } - let Ok((sequence, frame)) = voice::decode(&state.credentials, &bytes[..length]) - else { - continue; - }; + let (sequence, frame) = + match voice::decode(&state.credentials, &bytes[..length]) { + Ok(decoded) => decoded, + Err(error) => { + rejected += 1; + if rejected == 1 || rejected % 500 == 0 { + tracing::warn!(%error, rejected, "voice datagram rejected"); + } + continue; + } + }; if !state.replay.accept(sequence) || frame.opus.is_empty() { continue; } diff --git a/src/ui/update.rs b/src/ui/update.rs index 8354e25..3e0f1ac 100644 --- a/src/ui/update.rs +++ b/src/ui/update.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use vnox_client::audio::{IncomingVoice, PipelineConfig}; +use vnox_client::audio::{AudioStage, IncomingVoice, PipelineConfig}; use vnox_client::net::{ChatMsg, NetCommand, NetEvent, NetHandle, PresenceInfo}; use crate::ui::state::{ChatMessage, ConnState, UiState, VoiceUiState}; @@ -304,11 +304,13 @@ pub fn process_net_events(state: &mut AppState) { // Capture/playback runs independently from incoming control packets. sync_voice_audio(state); flush_encoded_voice(state); + report_audio_issues(state); + log_audio_stats(state); if let Some(pipeline) = state.audio.as_ref() { pipeline.sync_controls(vnox_client::audio::AudioSync { mic_enabled: state.ui.mic_enabled && !state.ui.deafen, vad_mode: state.ui.vad_mode, - vad_threshold: 40.0, + vad_threshold: 15.0, input_volume: 100.0, ptt_held: state.ui.ptt_held, output_volume: if state.ui.deafen { @@ -950,3 +952,66 @@ pub fn flush_encoded_voice(state: &mut AppState) { }); } } + +/// Surfaces capture and playback failures instead of leaving a "connected" room that is mute. +fn report_audio_issues(state: &mut AppState) { + let Some(pipeline) = state.audio.as_ref() else { + return; + }; + let issues = pipeline.take_issues(); + let dead = pipeline.is_dead(); + for issue in issues { + let message = match issue.stage { + AudioStage::Capture => { + crate::ui::i18n::Message::with("Microphone error: {e}", vec![("e", issue.message)]) + } + AudioStage::Playback => crate::ui::i18n::Message::with( + "Audio output error: {e}", + vec![("e", issue.message)], + ), + }; + state.ui.status_msg = Some(message); + } + if dead { + let reason = state + .ui + .status_msg + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(); + state.audio = None; + state.voice_channel = None; + state.ui.voice = VoiceUiState::Error(reason); + } +} + +/// Logs signal counters every ten seconds while a voice room is active, so a bug report can +/// tell whether the signal stopped at the microphone, the network or the speaker. +fn log_audio_stats(state: &AppState) { + thread_local! { + static LAST: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + let due = LAST.with(|last| { + let due = last + .get() + .is_none_or(|t| t.elapsed() >= std::time::Duration::from_secs(10)); + if due { + last.set(Some(std::time::Instant::now())); + } + due + }); + if !due { + return; + } + if let Some(pipeline) = state.audio.as_ref() { + let stats = pipeline.stats(); + tracing::info!( + captured = stats.captured_frames, + sent = stats.transmitted_frames, + received = stats.received_packets, + decoded = stats.decoded_frames, + input_peak = stats.input_peak, + "voice audio stats" + ); + } +} diff --git a/translations/ru.json b/translations/ru.json index 5618015..27afbee 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -216,6 +216,7 @@ "Reconnecting in {delay_secs}s (attempt {attempt})": "Повторное подключение через {delay_secs} с (попытка {attempt})", "Server error: {message}": "Ошибка сервера: {message}", "Microphone error: {e}": "Ошибка микрофона: {e}", + "Audio output error: {e}": "Ошибка вывода звука: {e}", "Cannot save audio settings: {error}": "Не удалось сохранить настройки звука: {error}", "Passphrases do not match": "Пароли не совпадают", "Passphrase must be at least 8 characters": "Пароль должен содержать минимум 8 символов", From 7d488faf59e18135305d573fee10139bb66abafd Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Sun, 25 Oct 2026 09:50:11 +0100 Subject: [PATCH 3/6] fix(client): derive voice status from a single state machine The status bar and voice page kept saying the room was being joined while the sidebar already reported a connected voice. Clicking the room you were already in set a free-text 'joining' message that nothing cleared, because the server does not answer a repeated join, and the voice page showed its joining label whenever the member list was empty. Joining is now an explicit voice state. The sidebar, the voice page and the status text all read one derived status; a repeated click only shows the room, and a server error cancels a pending join. --- src/main.rs | 16 ++-- src/net/session/handlers/voice.rs | 4 +- src/ui/bridge.rs | 7 +- src/ui/slint/main.slint | 2 + src/ui/slint/voice.slint | 3 +- src/ui/state/mod.rs | 2 + src/ui/state/types.rs | 4 + src/ui/state/voice.rs | 126 ++++++++++++++++++++++++++++++ src/ui/update.rs | 3 + 9 files changed, 153 insertions(+), 14 deletions(-) create mode 100644 src/ui/state/voice.rs diff --git a/src/main.rs b/src/main.rs index b2e4051..6f91cf8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -275,17 +275,19 @@ fn wire_signals(window: &MainWindow, state: &Rc>) { s.ui.channels .iter() .any(|c| c.id == id && c.kind == "voice"); - if is_voice { - s.ui.showing_voice = true; - s.ui.status_msg = Some("Joining voice channel…".into()); + let send_join = if is_voice { + s.ui.request_voice_join(&id) } else { s.ui.select_text_channel(id.clone()); - } + true + }; let net = s.net.clone(); drop(s); - tokio::spawn(async move { - net.send(NetCommand::JoinChannel { channel_id: id }).await; - }); + if send_join { + tokio::spawn(async move { + net.send(NetCommand::JoinChannel { channel_id: id }).await; + }); + } }); } diff --git a/src/net/session/handlers/voice.rs b/src/net/session/handlers/voice.rs index 9be37cc..a68bf2d 100644 --- a/src/net/session/handlers/voice.rs +++ b/src/net/session/handlers/voice.rs @@ -43,7 +43,7 @@ impl State { if source != state.endpoint { // A relay answering from another address looks exactly like silence. rejected += 1; - if rejected == 1 || rejected % 500 == 0 { + if rejected == 1 || rejected.is_multiple_of(500) { tracing::warn!( %source, expected = %state.endpoint, @@ -58,7 +58,7 @@ impl State { Ok(decoded) => decoded, Err(error) => { rejected += 1; - if rejected == 1 || rejected % 500 == 0 { + if rejected == 1 || rejected.is_multiple_of(500) { tracing::warn!(%error, rejected, "voice datagram rejected"); } continue; diff --git a/src/ui/bridge.rs b/src/ui/bridge.rs index e942eed..d0dc29c 100644 --- a/src/ui/bridge.rs +++ b/src/ui/bridge.rs @@ -338,10 +338,9 @@ pub fn sync_voice(window: &MainWindow, state: &UiState) { } else { window.set_voice_channel_name("".into()); } - window.set_voice_connected(matches!( - state.voice, - crate::ui::state::VoiceUiState::Active { .. } - )); + let status = state.voice_status(); + window.set_voice_connected(status == crate::ui::state::VoiceStatus::Connected); + window.set_voice_joining(status == crate::ui::state::VoiceStatus::Joining); let banner = match (state.local_speaking, state.remote_speaking) { (true, true) => crate::ui::i18n::text("You and others are talking"), diff --git a/src/ui/slint/main.slint b/src/ui/slint/main.slint index 7602e09..2878e2f 100644 --- a/src/ui/slint/main.slint +++ b/src/ui/slint/main.slint @@ -34,6 +34,7 @@ export component MainWindow inherits Window { private property pending-delete-id; in-out property active-dm-id; in-out property voice-connected; + in-out property voice-joining; in-out property members-visible: true; in-out property <[string]> input-devices; in-out property <[string]> output-devices; @@ -286,6 +287,7 @@ export component MainWindow inherits Window { deafened: root.deafened; channel-name: root.voice-channel-name; activity-banner: root.activity-banner; + joining: root.voice-joining; push-to-talk: root.vad-mode == 0; ptt-held(held) => { root.ptt-held(held); } toggle-mic => { root.toggle-mic(); } diff --git a/src/ui/slint/voice.slint b/src/ui/slint/voice.slint index 5ab5729..e830adf 100644 --- a/src/ui/slint/voice.slint +++ b/src/ui/slint/voice.slint @@ -8,6 +8,7 @@ export component VoicePanel inherits Rectangle { in-out property deafened; in-out property channel-name; in-out property activity-banner; + in property joining; in-out property push-to-talk; callback preview-screen(); callback ptt-held(bool); @@ -53,7 +54,7 @@ export component VoicePanel inherits Rectangle { Rectangle { horizontal-stretch: 1; } } } - if root.members.length == 0 : Text { text: @tr("Joining voice…"); color: Theme.text-muted; font-size: Math.max(11px, Theme.font-size * 0.875); } + if root.joining : Text { text: @tr("Joining voice…"); color: Theme.text-muted; font-size: Math.max(11px, Theme.font-size * 0.875); } Rectangle { vertical-stretch: 1; } } } diff --git a/src/ui/state/mod.rs b/src/ui/state/mod.rs index 86f317d..82ab5ca 100644 --- a/src/ui/state/mod.rs +++ b/src/ui/state/mod.rs @@ -2,9 +2,11 @@ mod default; mod models; mod navigation; mod types; +mod voice; pub use models::{Channel, ChatMessage, DmConversation, FriendState, GuildState, NodeBookmark}; pub use types::{ConnState, TrustPrompt, TrustPromptKind, UiState, VoiceUiState}; +pub use voice::VoiceStatus; use models::bookmark_label_from_address; diff --git a/src/ui/state/types.rs b/src/ui/state/types.rs index bdef68d..8e1c2c1 100644 --- a/src/ui/state/types.rs +++ b/src/ui/state/types.rs @@ -5,6 +5,10 @@ use std::time::Instant; pub enum VoiceUiState { #[default] Off, + /// A join was requested; audio is not running yet. + Joining { + channel: String, + }, Active { channel: String, }, diff --git a/src/ui/state/voice.rs b/src/ui/state/voice.rs new file mode 100644 index 0000000..a3404d5 --- /dev/null +++ b/src/ui/state/voice.rs @@ -0,0 +1,126 @@ +//! Voice connection status: one state machine feeds the status bar, the sidebar and the +//! voice page, so they can never disagree. +use super::{ConnState, UiState, VoiceUiState}; + +/// What the interface shows about the voice connection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VoiceStatus { + Off, + /// A join was requested and audio is not running yet. + Joining, + /// Audio is running in the voice room. + Connected, + Failed(String), +} + +impl UiState { + /// Derives the displayed voice status from the connection and voice state. + pub fn voice_status(&self) -> VoiceStatus { + if !matches!(self.conn, ConnState::Connected { .. }) { + return VoiceStatus::Off; + } + match &self.voice { + VoiceUiState::Off => VoiceStatus::Off, + VoiceUiState::Joining { .. } => VoiceStatus::Joining, + VoiceUiState::Active { .. } => VoiceStatus::Connected, + VoiceUiState::Error(message) => VoiceStatus::Failed(message.clone()), + } + } + + /// Handles a click on a voice channel. Returns true when a join request must be sent to + /// the server. Clicking the room you are already in only shows it: the server does not + /// answer a repeated join, so waiting for a reply would leave "joining" on screen forever. + pub fn request_voice_join(&mut self, channel_id: &str) -> bool { + self.showing_voice = true; + let current = match &self.voice { + VoiceUiState::Joining { channel } | VoiceUiState::Active { channel } => { + Some(channel.as_str()) + } + _ => None, + }; + if current == Some(channel_id) { + return false; + } + self.voice = VoiceUiState::Joining { + channel: channel_id.to_string(), + }; + true + } + + /// The server refused or lost a pending join: stop showing "joining". + pub fn cancel_pending_voice_join(&mut self) { + if matches!(self.voice, VoiceUiState::Joining { .. }) { + self.voice = VoiceUiState::Off; + self.voice_target = None; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn connected() -> UiState { + UiState { + conn: ConnState::Connected { + session_id: "s".into(), + node_name: "n".into(), + }, + ..UiState::default() + } + } + + #[test] + fn joining_becomes_connected_only_when_audio_is_active() { + let mut ui = connected(); + assert_eq!(ui.voice_status(), VoiceStatus::Off); + assert!(ui.request_voice_join("lounge")); + assert_eq!(ui.voice_status(), VoiceStatus::Joining); + ui.voice = VoiceUiState::Active { + channel: "lounge".into(), + }; + assert_eq!(ui.voice_status(), VoiceStatus::Connected); + } + + #[test] + fn clicking_the_current_room_never_restarts_joining() { + let mut ui = connected(); + ui.voice = VoiceUiState::Active { + channel: "lounge".into(), + }; + ui.showing_voice = false; + assert!(!ui.request_voice_join("lounge")); + assert!(ui.showing_voice); + assert_eq!(ui.voice_status(), VoiceStatus::Connected); + // A different room is a real join. + assert!(ui.request_voice_join("other")); + assert_eq!(ui.voice_status(), VoiceStatus::Joining); + } + + #[test] + fn retry_after_failure_and_cancel_of_a_refused_join() { + let mut ui = connected(); + ui.voice = VoiceUiState::Error("no mic".into()); + assert_eq!(ui.voice_status(), VoiceStatus::Failed("no mic".into())); + assert!(ui.request_voice_join("lounge")); + ui.cancel_pending_voice_join(); + assert_eq!(ui.voice_status(), VoiceStatus::Off); + // Cancelling does not touch an established connection. + ui.voice = VoiceUiState::Active { + channel: "lounge".into(), + }; + ui.cancel_pending_voice_join(); + assert_eq!(ui.voice_status(), VoiceStatus::Connected); + } + + #[test] + fn status_is_off_while_the_control_connection_is_down() { + let ui = UiState { + voice: VoiceUiState::Active { + channel: "lounge".into(), + }, + ..UiState::default() + }; + assert_eq!(ui.voice_status(), VoiceStatus::Off); + } +} diff --git a/src/ui/update.rs b/src/ui/update.rs index 3e0f1ac..0fc96ff 100644 --- a/src/ui/update.rs +++ b/src/ui/update.rs @@ -612,6 +612,7 @@ fn chat_handle(ui: &mut UiState, net: &NetHandle, event: NetEvent) { ui.scroll_bottom = true; } NetEvent::Error { message, .. } => { + ui.cancel_pending_voice_join(); if message.starts_with("media:") { crate::ui::media::on_error(&message); } @@ -931,6 +932,8 @@ pub fn sync_voice_audio(state: &mut AppState) { let active_voice = state.ui.voice_target.clone(); match active_voice { Some(id) => start_voice(state, &id), + // A pending join has no target until the server acknowledges it. + None if matches!(state.ui.voice, VoiceUiState::Joining { .. }) => {} None => stop_voice(state), } } From bf485720f51ec855b8796dd98ddb442a1b2560a7 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Sun, 25 Oct 2026 09:54:37 +0100 Subject: [PATCH 4/6] fix(client): keep the live voice session visible and stop orphaned microphones Switching from room A to room B replaced the voice state with a joining state while the microphone was still capturing in A, and a deleted room left the capture pipeline running with no visible state. A pending join is now stored separately, so the live session stays shown as connected until the server acknowledges the new room or refuses it. The capture pipeline is stopped whenever it no longer belongs to the current voice target, whatever a pending join is waiting for. --- src/ui/state/default.rs | 1 + src/ui/state/types.rs | 7 ++-- src/ui/state/voice.rs | 86 ++++++++++++++++++++++------------------- src/ui/update.rs | 42 +++++++++++++++++++- 4 files changed, 90 insertions(+), 46 deletions(-) diff --git a/src/ui/state/default.rs b/src/ui/state/default.rs index cb68d9f..9cdf67f 100644 --- a/src/ui/state/default.rs +++ b/src/ui/state/default.rs @@ -21,6 +21,7 @@ impl Default for UiState { browsing_dms: false, showing_voice: false, voice_target: None, + pending_voice_join: None, vault_enabled: false, scroll_bottom: false, voice: VoiceUiState::Off, diff --git a/src/ui/state/types.rs b/src/ui/state/types.rs index 8e1c2c1..7817458 100644 --- a/src/ui/state/types.rs +++ b/src/ui/state/types.rs @@ -5,10 +5,6 @@ use std::time::Instant; pub enum VoiceUiState { #[default] Off, - /// A join was requested; audio is not running yet. - Joining { - channel: String, - }, Active { channel: String, }, @@ -64,6 +60,9 @@ pub struct UiState { pub browsing_dms: bool, pub showing_voice: bool, pub voice_target: Option, + /// Voice room whose join was requested and not yet acknowledged. The current session, if + /// any, keeps running and keeps being shown until the server answers. + pub pending_voice_join: Option, pub vault_enabled: bool, pub scroll_bottom: bool, diff --git a/src/ui/state/voice.rs b/src/ui/state/voice.rs index a3404d5..2299853 100644 --- a/src/ui/state/voice.rs +++ b/src/ui/state/voice.rs @@ -14,45 +14,40 @@ pub enum VoiceStatus { } impl UiState { - /// Derives the displayed voice status from the connection and voice state. + /// Derives the displayed voice status from the connection and voice state. A running + /// session is always shown as connected, also while a switch to another room is pending, + /// because its microphone is still live. pub fn voice_status(&self) -> VoiceStatus { if !matches!(self.conn, ConnState::Connected { .. }) { return VoiceStatus::Off; } match &self.voice { - VoiceUiState::Off => VoiceStatus::Off, - VoiceUiState::Joining { .. } => VoiceStatus::Joining, VoiceUiState::Active { .. } => VoiceStatus::Connected, + _ if self.pending_voice_join.is_some() => VoiceStatus::Joining, + VoiceUiState::Off => VoiceStatus::Off, VoiceUiState::Error(message) => VoiceStatus::Failed(message.clone()), } } /// Handles a click on a voice channel. Returns true when a join request must be sent to - /// the server. Clicking the room you are already in only shows it: the server does not - /// answer a repeated join, so waiting for a reply would leave "joining" on screen forever. + /// the server. Clicking the room you are already in (or already joining) only shows it: + /// the server does not answer a repeated join, so waiting for a reply would leave + /// "joining" on screen forever. pub fn request_voice_join(&mut self, channel_id: &str) -> bool { self.showing_voice = true; - let current = match &self.voice { - VoiceUiState::Joining { channel } | VoiceUiState::Active { channel } => { - Some(channel.as_str()) - } - _ => None, - }; - if current == Some(channel_id) { + let in_room = + matches!(&self.voice, VoiceUiState::Active { channel } if channel == channel_id); + if in_room || self.pending_voice_join.as_deref() == Some(channel_id) { return false; } - self.voice = VoiceUiState::Joining { - channel: channel_id.to_string(), - }; + self.pending_voice_join = Some(channel_id.to_string()); true } - /// The server refused or lost a pending join: stop showing "joining". + /// The server refused or lost a pending join: stop showing "joining". The current session + /// is left untouched. pub fn cancel_pending_voice_join(&mut self) { - if matches!(self.voice, VoiceUiState::Joining { .. }) { - self.voice = VoiceUiState::Off; - self.voice_target = None; - } + self.pending_voice_join = None; } } @@ -70,55 +65,66 @@ mod tests { } } + fn active(channel: &str) -> VoiceUiState { + VoiceUiState::Active { + channel: channel.into(), + } + } + #[test] fn joining_becomes_connected_only_when_audio_is_active() { let mut ui = connected(); assert_eq!(ui.voice_status(), VoiceStatus::Off); assert!(ui.request_voice_join("lounge")); assert_eq!(ui.voice_status(), VoiceStatus::Joining); - ui.voice = VoiceUiState::Active { - channel: "lounge".into(), - }; + // The acknowledgement clears the request; audio start makes it connected. + ui.pending_voice_join = None; + ui.voice = active("lounge"); assert_eq!(ui.voice_status(), VoiceStatus::Connected); } #[test] - fn clicking_the_current_room_never_restarts_joining() { + fn clicking_the_current_or_pending_room_never_restarts_joining() { let mut ui = connected(); - ui.voice = VoiceUiState::Active { - channel: "lounge".into(), - }; + ui.voice = active("lounge"); ui.showing_voice = false; assert!(!ui.request_voice_join("lounge")); assert!(ui.showing_voice); assert_eq!(ui.voice_status(), VoiceStatus::Connected); - // A different room is a real join. assert!(ui.request_voice_join("other")); - assert_eq!(ui.voice_status(), VoiceStatus::Joining); + assert!(!ui.request_voice_join("other")); } #[test] - fn retry_after_failure_and_cancel_of_a_refused_join() { + fn switching_rooms_keeps_showing_the_live_session_until_acknowledged() { + let mut ui = connected(); + ui.voice = active("a"); + ui.voice_target = Some("a".into()); + assert!(ui.request_voice_join("b")); + // The microphone is still live in A, so the UI must say so. + assert_eq!(ui.voice, active("a")); + assert_eq!(ui.voice_status(), VoiceStatus::Connected); + // A refused join leaves the session in A alone. + ui.cancel_pending_voice_join(); + assert_eq!(ui.voice, active("a")); + assert_eq!(ui.voice_status(), VoiceStatus::Connected); + } + + #[test] + fn retry_after_failure_shows_joining_and_cancel_returns_to_failure() { let mut ui = connected(); ui.voice = VoiceUiState::Error("no mic".into()); assert_eq!(ui.voice_status(), VoiceStatus::Failed("no mic".into())); assert!(ui.request_voice_join("lounge")); + assert_eq!(ui.voice_status(), VoiceStatus::Joining); ui.cancel_pending_voice_join(); - assert_eq!(ui.voice_status(), VoiceStatus::Off); - // Cancelling does not touch an established connection. - ui.voice = VoiceUiState::Active { - channel: "lounge".into(), - }; - ui.cancel_pending_voice_join(); - assert_eq!(ui.voice_status(), VoiceStatus::Connected); + assert_eq!(ui.voice_status(), VoiceStatus::Failed("no mic".into())); } #[test] fn status_is_off_while_the_control_connection_is_down() { let ui = UiState { - voice: VoiceUiState::Active { - channel: "lounge".into(), - }, + voice: active("lounge"), ..UiState::default() }; assert_eq!(ui.voice_status(), VoiceStatus::Off); diff --git a/src/ui/update.rs b/src/ui/update.rs index 0fc96ff..4ae73b4 100644 --- a/src/ui/update.rs +++ b/src/ui/update.rs @@ -453,6 +453,9 @@ fn chat_handle(ui: &mut UiState, net: &NetHandle, event: NetEvent) { } if kind == "voice" { ui.voice_target = Some(channel_id.clone()); + if ui.pending_voice_join.as_deref() == Some(channel_id.as_str()) { + ui.pending_voice_join = None; + } // A newly acknowledged join is also an explicit retry after an audio error. if matches!(ui.voice, VoiceUiState::Error(_)) { ui.voice = VoiceUiState::Off; @@ -930,10 +933,16 @@ pub fn sync_voice_audio(state: &mut AppState) { return; } let active_voice = state.ui.voice_target.clone(); + if pipeline_should_stop( + active_voice.as_deref(), + state.audio.as_ref().and(state.voice_channel.as_deref()), + ) { + stop_voice(state); + } match active_voice { Some(id) => start_voice(state, &id), - // A pending join has no target until the server acknowledges it. - None if matches!(state.ui.voice, VoiceUiState::Joining { .. }) => {} + // No target (room deleted, revoked, left): the capture pipeline must not outlive it, + // whatever a pending join is waiting for. None => stop_voice(state), } } @@ -1018,3 +1027,32 @@ fn log_audio_stats(state: &AppState) { ); } } + +/// A live capture pipeline must belong to the room the client is currently targeting. When the +/// target disappears (room deleted, access revoked) or moves elsewhere, the microphone stops +/// immediately, independent of what a pending join is waiting for. +fn pipeline_should_stop(target: Option<&str>, live_channel: Option<&str>) -> bool { + live_channel.is_some() && live_channel != target +} + +#[cfg(test)] +mod voice_pipeline_tests { + use super::pipeline_should_stop; + + #[test] + fn deleted_room_stops_the_microphone() { + assert!(pipeline_should_stop(None, Some("a"))); + } + + #[test] + fn moving_to_another_room_stops_the_old_pipeline() { + assert!(pipeline_should_stop(Some("b"), Some("a"))); + } + + #[test] + fn matching_room_or_no_pipeline_is_left_alone() { + assert!(!pipeline_should_stop(Some("a"), Some("a"))); + assert!(!pipeline_should_stop(None, None)); + assert!(!pipeline_should_stop(Some("a"), None)); + } +} From eed3e28c638eda08c91b6bc4d4105103fb66acae Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Sun, 25 Oct 2026 10:01:38 +0100 Subject: [PATCH 5/6] feat(client): synthesized notification sounds with settings Short tones for joining and leaving a voice room, other participants joining and leaving, new messages outside the viewed chat, mentions, microphone mute and unmute, deafen and undeafen, server connection and loss, and errors. The tones are generated at runtime (rising motifs for arrival or enable, falling for leave or disable) with an attack and a fade-out so they never click; no third-party audio is bundled. Inside a voice room the sound is mixed into the open playback stream, otherwise a short one-shot stream is opened on the selected output device off the UI thread, capped at two at a time. While deafened nothing plays except the undeafen sound. Settings: master switch, volume and separate switches for voice room, messages and mentions, stored with the audio preferences, with a new settings section and a test button. --- docs/THIRD_PARTY.md | 12 + src/audio/mod.rs | 1 + src/audio/sounds/mod.rs | 247 +++++++++++++++++++++ src/audio/sounds/player.rs | 155 +++++++++++++ src/audio/sounds/synth.rs | 234 +++++++++++++++++++ src/main.rs | 43 +++- src/ui/bridge.rs | 5 + src/ui/mod.rs | 1 + src/ui/preferences.rs | 8 + src/ui/slint/main.slint | 14 ++ src/ui/slint/settings.slint | 17 ++ src/ui/slint/settings/sounds.slint | 86 +++++++ src/ui/sounds.rs | 217 ++++++++++++++++++ src/ui/state/default.rs | 1 + src/ui/state/types.rs | 1 + src/ui/update.rs | 14 ++ translations/en/LC_MESSAGES/vnox-client.po | 36 +++ translations/ru.json | 16 +- translations/ru/LC_MESSAGES/vnox-client.po | 38 +++- 19 files changed, 1142 insertions(+), 4 deletions(-) create mode 100644 docs/THIRD_PARTY.md create mode 100644 src/audio/sounds/mod.rs create mode 100644 src/audio/sounds/player.rs create mode 100644 src/audio/sounds/synth.rs create mode 100644 src/ui/slint/settings/sounds.slint create mode 100644 src/ui/sounds.rs diff --git a/docs/THIRD_PARTY.md b/docs/THIRD_PARTY.md new file mode 100644 index 0000000..03362d8 --- /dev/null +++ b/docs/THIRD_PARTY.md @@ -0,0 +1,12 @@ +# Third-party material + +## Notification sounds + +The notification sounds are not recordings. They are generated at runtime by +`src/audio/sounds/synth.rs` (sine tones with two quiet harmonics, a short attack, an +exponential decay and a fade-out). No third-party audio files, samples or presets are bundled, +so they carry no extra attribution or licence: they are part of VNOX-Client under GPL-3.0. + +## Interface icons + +Icon sets and their licences (Lucide, ISC) are documented with the icon assets. diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 97b9b4e..1f272eb 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -3,6 +3,7 @@ pub mod devices; pub mod diagnostics; mod pipeline; mod processing; +pub mod sounds; pub use devices::{DeviceEntry, DeviceLists}; pub use diagnostics::{AudioIssue, AudioStage, AudioStats}; diff --git a/src/audio/sounds/mod.rs b/src/audio/sounds/mod.rs new file mode 100644 index 0000000..88fd4ce --- /dev/null +++ b/src/audio/sounds/mod.rs @@ -0,0 +1,247 @@ +//! Notification sounds: which events make a sound, settings, and the tones themselves. +//! +//! The tones are synthesized (see [`synth`]), so the repository carries no third-party audio +//! and no extra licence. Event selection is pure logic and is unit tested; playback lives in +//! [`player`]. + +mod player; +pub mod synth; + +pub use player::SoundPlayer; + +use serde::{Deserialize, Serialize}; + +/// Everything the client can announce with a sound. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SoundEvent { + /// You entered a voice room. + VoiceJoined, + /// You left a voice room. + VoiceLeft, + PeerJoined, + PeerLeft, + /// A new message in a channel or conversation you are not looking at. + Message, + /// A message that mentions you. + Mention, + MicMuted, + MicUnmuted, + Deafened, + Undeafened, + Connected, + ConnectionLost, + Error, +} + +/// The settings group an event belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SoundCategory { + VoiceRoom, + Messages, + Mentions, + /// Connection and error sounds follow only the master switch. + System, +} + +impl SoundEvent { + pub fn category(self) -> SoundCategory { + match self { + Self::VoiceJoined + | Self::VoiceLeft + | Self::PeerJoined + | Self::PeerLeft + | Self::MicMuted + | Self::MicUnmuted + | Self::Deafened + | Self::Undeafened => SoundCategory::VoiceRoom, + Self::Message => SoundCategory::Messages, + Self::Mention => SoundCategory::Mentions, + Self::Connected | Self::ConnectionLost | Self::Error => SoundCategory::System, + } + } +} + +/// User-facing sound preferences, persisted with the audio settings. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct SoundSettings { + /// Master switch. + pub enabled: bool, + /// 0 to 100. + pub volume: f32, + pub voice_room: bool, + pub messages: bool, + pub mentions: bool, +} + +impl Default for SoundSettings { + fn default() -> Self { + Self { + enabled: true, + volume: 60.0, + voice_room: true, + messages: true, + mentions: true, + } + } +} + +impl SoundSettings { + /// Linear gain for the volume slider (perceptual curve, 0 is silent). + pub fn gain(&self) -> f32 { + let v = (self.volume / 100.0).clamp(0.0, 1.0); + v * v + } +} + +/// Decides whether `event` should be heard right now. `deafened` is the local deafen state. +/// +/// While deafened nothing plays except the sound announcing that you can hear again. +pub fn select(event: SoundEvent, settings: &SoundSettings, deafened: bool) -> Option { + if !settings.enabled || settings.gain() <= 0.0 { + return None; + } + if deafened && event != SoundEvent::Undeafened { + return None; + } + let allowed = match event.category() { + SoundCategory::VoiceRoom => settings.voice_room, + SoundCategory::Messages => settings.messages, + SoundCategory::Mentions => settings.mentions, + SoundCategory::System => true, + }; + allowed.then_some(event) +} + +/// Picks the sound for an incoming message, or `None` for silence. +/// +/// Your own messages are silent; a mention is announced even in the channel you are viewing +/// (it is addressed to you), an ordinary message only when you are looking elsewhere. +pub fn message_event(own: bool, mentions_me: bool, viewing_this_chat: bool) -> Option { + if own { + None + } else if mentions_me { + Some(SoundEvent::Mention) + } else if viewing_this_chat { + None + } else { + Some(SoundEvent::Message) + } +} + +/// True when `content` addresses this user: `@nickname` (case-insensitive, whole word), +/// `@everyone` or `@here`. +pub fn is_mention(content: &str, nickname: &str) -> bool { + let lowered = content.to_lowercase(); + let nick = nickname.trim().to_lowercase(); + lowered.match_indices('@').any(|(at, _)| { + let word: String = lowered[at + 1..] + .chars() + .take_while(|c| c.is_alphanumeric() || matches!(c, '_' | '-' | '.')) + .collect(); + let word = word.trim_end_matches('.'); + !word.is_empty() + && (word == "everyone" || word == "here" || (!nick.is_empty() && word == nick)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn master_switch_and_zero_volume_silence_everything() { + let off = SoundSettings { + enabled: false, + ..Default::default() + }; + assert_eq!(select(SoundEvent::Error, &off, false), None); + let quiet = SoundSettings { + volume: 0.0, + ..Default::default() + }; + assert_eq!(select(SoundEvent::Message, &quiet, false), None); + } + + #[test] + fn categories_are_switched_independently() { + let settings = SoundSettings { + voice_room: false, + messages: true, + mentions: false, + ..Default::default() + }; + assert_eq!(select(SoundEvent::PeerJoined, &settings, false), None); + assert_eq!(select(SoundEvent::MicMuted, &settings, false), None); + assert_eq!(select(SoundEvent::Mention, &settings, false), None); + assert_eq!( + select(SoundEvent::Message, &settings, false), + Some(SoundEvent::Message) + ); + // Connection and error sounds only depend on the master switch. + assert_eq!( + select(SoundEvent::ConnectionLost, &settings, false), + Some(SoundEvent::ConnectionLost) + ); + } + + #[test] + fn deafen_silences_everything_except_the_undeafen_sound() { + let settings = SoundSettings::default(); + for event in [ + SoundEvent::Message, + SoundEvent::Mention, + SoundEvent::PeerJoined, + SoundEvent::Deafened, + SoundEvent::Error, + SoundEvent::ConnectionLost, + ] { + assert_eq!(select(event, &settings, true), None, "{event:?}"); + } + assert_eq!( + select(SoundEvent::Undeafened, &settings, true), + Some(SoundEvent::Undeafened) + ); + assert_eq!( + select(SoundEvent::Undeafened, &settings, false), + Some(SoundEvent::Undeafened) + ); + } + + #[test] + fn message_sound_follows_focus_ownership_and_mentions() { + assert_eq!(message_event(true, true, false), None); + assert_eq!(message_event(false, false, true), None); + assert_eq!( + message_event(false, false, false), + Some(SoundEvent::Message) + ); + assert_eq!(message_event(false, true, true), Some(SoundEvent::Mention)); + assert_eq!(message_event(false, true, false), Some(SoundEvent::Mention)); + } + + #[test] + fn mentions_match_whole_words_only() { + assert!(is_mention("hey @Alice, look", "alice")); + assert!(is_mention("@alice", "Alice")); + assert!(is_mention("ping @everyone", "bob")); + assert!(is_mention("@here meeting", "bob")); + assert!(is_mention("thanks @alice.", "alice")); + assert!(!is_mention("hey @alicia", "alice")); + assert!(!is_mention("mail alice@example.com", "example")); + assert!(!is_mention("alice without at", "alice")); + assert!(!is_mention("@", "")); + } + + #[test] + fn settings_deserialize_with_defaults_and_clamp_volume_gain() { + let parsed: SoundSettings = serde_json::from_str(r#"{"volume":100.0}"#).unwrap(); + assert!(parsed.enabled && parsed.voice_room && parsed.messages && parsed.mentions); + assert!((parsed.gain() - 1.0).abs() < f32::EPSILON); + let loud = SoundSettings { + volume: 900.0, + ..Default::default() + }; + assert!((loud.gain() - 1.0).abs() < f32::EPSILON); + } +} diff --git a/src/audio/sounds/player.rs b/src/audio/sounds/player.rs new file mode 100644 index 0000000..d5914ec --- /dev/null +++ b/src/audio/sounds/player.rs @@ -0,0 +1,155 @@ +//! Plays a notification sound without blocking the UI thread. +//! +//! In a voice room the sound is mixed into the already open playback stream. Otherwise a +//! short-lived stream is opened on the selected output device in its own thread and closed +//! when the sound ends. A small cap on concurrent streams keeps a burst of events from +//! piling up devices. + +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use cpal::traits::{DeviceTrait, StreamTrait}; +use tracing::warn; + +use super::{SoundEvent, synth}; +use crate::audio::{ + AudioPipeline, + config::{AudioConfig, OPUS_RATE, resample_rate}, + devices, +}; + +const MAX_CONCURRENT_STREAMS: usize = 2; +/// Silence appended after the sound so the device drains before the stream is dropped. +const TAIL: Duration = Duration::from_millis(80); + +/// Fire-and-forget sound playback. +#[derive(Clone, Default)] +pub struct SoundPlayer { + active_streams: Arc, +} + +impl SoundPlayer { + /// Plays `event` at `gain` (0.0 to 1.0). With a running `pipeline` the sound joins its + /// stream; otherwise a short stream is opened on `output_device_id` (empty is the default). + pub fn play( + &self, + event: SoundEvent, + gain: f32, + pipeline: Option<&AudioPipeline>, + output_device_id: &str, + ) { + let samples = scaled(synth::render(event), gain); + if let Some(pipeline) = pipeline { + pipeline.play_effect(&samples); + return; + } + if self.active_streams.fetch_add(1, Ordering::AcqRel) >= MAX_CONCURRENT_STREAMS { + self.active_streams.fetch_sub(1, Ordering::AcqRel); + return; + } + let counter = self.active_streams.clone(); + let device_id = output_device_id.to_string(); + let spawned = std::thread::Builder::new() + .name("vnox-sound".into()) + .spawn(move || { + if let Err(error) = play_once(&samples, &device_id) { + warn!(%error, "cannot play notification sound"); + } + counter.fetch_sub(1, Ordering::AcqRel); + }); + if let Err(error) = spawned { + self.active_streams.fetch_sub(1, Ordering::AcqRel); + warn!(%error, "cannot start the sound thread"); + } + } +} + +fn scaled(samples: &[f32], gain: f32) -> Vec { + let gain = gain.clamp(0.0, 1.0); + samples.iter().map(|s| s * gain).collect() +} + +/// Converts mono 48 kHz samples to an interleaved buffer for the device format. +pub(crate) fn to_device_buffer(mono_48k: &[f32], device_rate: u32, channels: usize) -> Vec { + let mono = if device_rate == OPUS_RATE { + mono_48k.to_vec() + } else { + resample_rate(mono_48k, OPUS_RATE, device_rate) + }; + let channels = channels.max(1); + let mut out = Vec::with_capacity(mono.len() * channels); + for sample in mono { + out.extend(std::iter::repeat_n(sample, channels)); + } + out +} + +fn play_once(mono_48k: &[f32], device_id: &str) -> Result<()> { + let device = devices::open_output(device_id)?; + let config = AudioConfig::find(&device, false)?; + let channels = usize::from(config.stream.channels); + let buffer = to_device_buffer(mono_48k, config.sample_rate, channels); + let duration = + Duration::from_secs_f32(buffer.len() as f32 / channels as f32 / config.sample_rate as f32) + + TAIL; + let mut position = 0usize; + let stream = device + .build_output_stream( + config.stream, + move |out: &mut [f32], _| { + let n = out.len().min(buffer.len().saturating_sub(position)); + out[..n].copy_from_slice(&buffer[position..position + n]); + out[n..].fill(0.0); + position += n; + }, + |error| warn!(%error, "notification sound stream"), + None, + ) + .map_err(|e| anyhow!("{e}")) + .context("cannot open the sound stream")?; + stream.play().context("cannot start the sound stream")?; + std::thread::sleep(duration); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn device_buffer_duplicates_mono_across_channels() { + let buffer = to_device_buffer(&[0.1, 0.2], OPUS_RATE, 2); + assert_eq!(buffer, vec![0.1, 0.1, 0.2, 0.2]); + } + + #[test] + fn device_buffer_resamples_to_the_device_rate() { + let buffer = to_device_buffer(&vec![0.5; 4800], 24_000, 1); + assert_eq!(buffer.len(), 2400); + } + + #[test] + fn gain_scales_and_clamps() { + assert_eq!(scaled(&[0.5, -0.5], 0.5), vec![0.25, -0.25]); + assert_eq!(scaled(&[0.5], 7.0), vec![0.5]); + assert_eq!(scaled(&[0.5], -1.0), vec![0.0]); + } + + #[test] + fn concurrent_one_shot_streams_are_capped() { + let player = SoundPlayer::default(); + player + .active_streams + .store(MAX_CONCURRENT_STREAMS, Ordering::SeqCst); + // Over the cap the call returns immediately and leaves the counter unchanged. + player.play(SoundEvent::Message, 1.0, None, ""); + assert_eq!( + player.active_streams.load(Ordering::SeqCst), + MAX_CONCURRENT_STREAMS + ); + } +} diff --git a/src/audio/sounds/synth.rs b/src/audio/sounds/synth.rs new file mode 100644 index 0000000..dcf1c18 --- /dev/null +++ b/src/audio/sounds/synth.rs @@ -0,0 +1,234 @@ +//! Tone generator for the notification sounds. +//! +//! Every sound is a short motif of soft bell-like notes: a sine with two quiet harmonics, a +//! 6 ms attack, an exponential decay and a final fade to zero, so playback never clicks. +//! Rising motifs mean "arrived or enabled", falling motifs mean "left or disabled". +//! The output is mono 48 kHz f32 with a peak of 0.5 before the user volume is applied. + +use std::collections::HashMap; +use std::f32::consts::TAU; +use std::sync::OnceLock; + +use super::SoundEvent; +use crate::audio::config::OPUS_RATE; + +const ATTACK_MS: f32 = 6.0; +const RELEASE_MS: f32 = 12.0; +const PEAK: f32 = 0.5; + +/// One note of a motif. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Note { + pub hz: f32, + pub start_ms: f32, + pub duration_ms: f32, + pub amplitude: f32, +} + +const fn note(hz: f32, start_ms: f32, duration_ms: f32, amplitude: f32) -> Note { + Note { + hz, + start_ms, + duration_ms, + amplitude, + } +} + +// Pitches in Hz (equal temperament, A4 = 440). +const C5: f32 = 523.25; +const E5: f32 = 659.25; +const G5: f32 = 783.99; +const A5: f32 = 880.0; +const B5: f32 = 987.77; +const C6: f32 = 1046.5; +const E6: f32 = 1318.5; +const A4: f32 = 440.0; +const E4: f32 = 329.63; +const D4: f32 = 293.66; + +/// The motif for an event. +pub fn motif(event: SoundEvent) -> Vec { + match event { + SoundEvent::VoiceJoined => vec![note(C5, 0.0, 170.0, 0.9), note(G5, 110.0, 260.0, 1.0)], + SoundEvent::VoiceLeft => vec![note(G5, 0.0, 170.0, 0.9), note(C5, 110.0, 260.0, 1.0)], + SoundEvent::PeerJoined => vec![note(E5, 0.0, 110.0, 0.55), note(A5, 80.0, 190.0, 0.6)], + SoundEvent::PeerLeft => vec![note(A5, 0.0, 110.0, 0.55), note(E5, 80.0, 190.0, 0.6)], + SoundEvent::Message => vec![note(B5, 0.0, 230.0, 0.7)], + SoundEvent::Mention => vec![ + note(E6, 0.0, 120.0, 0.9), + note(B5, 100.0, 120.0, 0.8), + note(E6, 200.0, 280.0, 1.0), + ], + SoundEvent::MicMuted => vec![note(A5, 0.0, 70.0, 0.6), note(E5, 60.0, 110.0, 0.6)], + SoundEvent::MicUnmuted => vec![note(E5, 0.0, 70.0, 0.6), note(A5, 60.0, 110.0, 0.6)], + SoundEvent::Deafened => vec![ + note(G5, 0.0, 80.0, 0.6), + note(E5, 70.0, 80.0, 0.6), + note(C5, 140.0, 140.0, 0.6), + ], + SoundEvent::Undeafened => vec![ + note(C5, 0.0, 80.0, 0.6), + note(E5, 70.0, 80.0, 0.6), + note(G5, 140.0, 140.0, 0.6), + ], + SoundEvent::Connected => vec![ + note(C5, 0.0, 120.0, 0.7), + note(E5, 90.0, 120.0, 0.7), + note(C6, 180.0, 300.0, 0.8), + ], + SoundEvent::ConnectionLost => vec![note(A4, 0.0, 160.0, 0.8), note(E4, 130.0, 320.0, 0.8)], + SoundEvent::Error => vec![note(E4, 0.0, 120.0, 0.9), note(D4, 150.0, 220.0, 0.9)], + } +} + +fn sample_count(ms: f32) -> usize { + (ms * OPUS_RATE as f32 / 1000.0).round() as usize +} + +/// Renders a motif to mono 48 kHz samples with a peak of [`PEAK`]. +pub fn render_motif(notes: &[Note]) -> Vec { + let total_ms = notes + .iter() + .map(|n| n.start_ms + n.duration_ms) + .fold(0.0f32, f32::max); + // A short silent tail keeps the last release inside the buffer. + let mut out = vec![0.0f32; sample_count(total_ms) + sample_count(RELEASE_MS)]; + let rate = OPUS_RATE as f32; + for n in notes { + let start = sample_count(n.start_ms); + let length = sample_count(n.duration_ms); + let attack = sample_count(ATTACK_MS).max(1); + let release = sample_count(RELEASE_MS).max(1).min(length); + let tau = n.duration_ms / 1000.0 * 0.4; + for i in 0..length { + let t = i as f32 / rate; + let phase = TAU * n.hz * t; + let tone = phase.sin() + 0.25 * (2.0 * phase).sin() + 0.08 * (3.0 * phase).sin(); + let mut envelope = (-t / tau).exp(); + if i < attack { + envelope *= i as f32 / attack as f32; + } + if i + release > length { + envelope *= (length - i) as f32 / release as f32; + } + if let Some(slot) = out.get_mut(start + i) { + *slot += tone * envelope * n.amplitude; + } + } + } + let peak = out.iter().fold(0.0f32, |p, s| p.max(s.abs())); + if peak > 0.0 { + let scale = PEAK / peak; + for s in &mut out { + *s *= scale; + } + } + out +} + +/// Cached rendering of an event's sound. +pub fn render(event: SoundEvent) -> &'static [f32] { + static CACHE: OnceLock>> = OnceLock::new(); + const ALL: [SoundEvent; 13] = [ + SoundEvent::VoiceJoined, + SoundEvent::VoiceLeft, + SoundEvent::PeerJoined, + SoundEvent::PeerLeft, + SoundEvent::Message, + SoundEvent::Mention, + SoundEvent::MicMuted, + SoundEvent::MicUnmuted, + SoundEvent::Deafened, + SoundEvent::Undeafened, + SoundEvent::Connected, + SoundEvent::ConnectionLost, + SoundEvent::Error, + ]; + CACHE + .get_or_init(|| { + ALL.iter() + .map(|&event| (event, render_motif(&motif(event)))) + .collect() + }) + .get(&event) + .map(Vec::as_slice) + .unwrap_or(&[]) +} + +#[cfg(test)] +mod tests { + use super::*; + + const ALL: [SoundEvent; 13] = [ + SoundEvent::VoiceJoined, + SoundEvent::VoiceLeft, + SoundEvent::PeerJoined, + SoundEvent::PeerLeft, + SoundEvent::Message, + SoundEvent::Mention, + SoundEvent::MicMuted, + SoundEvent::MicUnmuted, + SoundEvent::Deafened, + SoundEvent::Undeafened, + SoundEvent::Connected, + SoundEvent::ConnectionLost, + SoundEvent::Error, + ]; + + #[test] + fn every_event_has_a_short_bounded_nonsilent_sound() { + for event in ALL { + let samples = render(event); + assert!(!samples.is_empty(), "{event:?}"); + // Under 0.8 s and within the peak ceiling. + assert!(samples.len() < OPUS_RATE as usize * 8 / 10, "{event:?}"); + let peak = samples.iter().fold(0.0f32, |p, s| p.max(s.abs())); + assert!(peak > 0.3 && peak <= PEAK + 1e-4, "{event:?} peak {peak}"); + assert!(samples.iter().all(|s| s.is_finite())); + } + } + + #[test] + fn sounds_start_and_end_at_zero_so_they_cannot_click() { + for event in ALL { + let samples = render(event); + assert!(samples[0].abs() < 1e-3, "{event:?} start"); + assert!(samples[samples.len() - 1].abs() < 1e-3, "{event:?} end"); + } + } + + #[test] + fn join_motifs_rise_and_leave_motifs_fall() { + let rises = |event| { + let m = motif(event); + m[m.len() - 1].hz > m[0].hz + }; + assert!(rises(SoundEvent::VoiceJoined)); + assert!(!rises(SoundEvent::VoiceLeft)); + assert!(rises(SoundEvent::PeerJoined)); + assert!(!rises(SoundEvent::PeerLeft)); + assert!(rises(SoundEvent::MicUnmuted)); + assert!(!rises(SoundEvent::MicMuted)); + assert!(rises(SoundEvent::Undeafened)); + assert!(!rises(SoundEvent::Deafened)); + } + + #[test] + fn rendered_tone_has_the_requested_pitch() { + let samples = render_motif(&[note(A5, 0.0, 300.0, 1.0)]); + // Count rising zero crossings in the stable part of the note. + let window = &samples[sample_count(40.0)..sample_count(240.0)]; + let crossings = window + .windows(2) + .filter(|w| w[0] <= 0.0 && w[1] > 0.0) + .count(); + let seconds = window.len() as f32 / OPUS_RATE as f32; + let measured = crossings as f32 / seconds; + assert!((measured - A5).abs() < 25.0, "measured {measured} Hz"); + } + + #[test] + fn empty_motif_renders_silence_without_panicking() { + assert!(render_motif(&[]).iter().all(|s| *s == 0.0)); + } +} diff --git a/src/main.rs b/src/main.rs index 6f91cf8..1a30750 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,7 @@ use tracing::info; use tracing_subscriber::EnvFilter; use vnox_client::audio::AudioPipeline; use vnox_client::audio::PipelineConfig; +use vnox_client::audio::sounds::SoundEvent; use vnox_client::identity; use vnox_client::net; use vnox_client::net::NetCommand; @@ -154,7 +155,7 @@ fn refresh_devices(window: &MainWindow, ui: &mut UiState) { .map(|(index, label)| { // Position 0 is always the system default. slint::SharedString::from(if index == 0 { - crate::ui::i18n::text("System default") + crate::ui::i18n::text("Default audio device") } else { label }) @@ -570,11 +571,41 @@ fn wire_signals(window: &MainWindow, state: &Rc>) { }); } + { + let state = state.clone(); + window.on_sound_settings_changed(move |enabled, volume, voice_room, messages, mentions| { + let mut s = state.borrow_mut(); + s.ui.sounds = vnox_client::audio::sounds::SoundSettings { + enabled, + volume: volume.clamp(0.0, 100.0), + voice_room, + messages, + mentions, + }; + if let Err(error) = crate::ui::preferences::save(&s.ui) { + s.ui.status_msg = Some(ui::i18n::Message::with( + "Cannot save audio settings: {error}", + vec![("error", error.to_string())], + )); + } + }); + } + { + let state = state.clone(); + window.on_preview_sound(move || crate::ui::sounds::preview(&state.borrow())); + } + { let state = state.clone(); window.on_toggle_mic(move || { let mut s = state.borrow_mut(); s.ui.mic_enabled = !s.ui.mic_enabled; + let event = if s.ui.mic_enabled { + SoundEvent::MicUnmuted + } else { + SoundEvent::MicMuted + }; + crate::ui::sounds::play(&s, event); }); } @@ -583,6 +614,12 @@ fn wire_signals(window: &MainWindow, state: &Rc>) { window.on_toggle_deafen(move || { let mut s = state.borrow_mut(); s.ui.deafen = !s.ui.deafen; + let event = if s.ui.deafen { + SoundEvent::Deafened + } else { + SoundEvent::Undeafened + }; + crate::ui::sounds::play(&s, event); }); } @@ -591,7 +628,11 @@ fn wire_signals(window: &MainWindow, state: &Rc>) { window.on_leave_voice(move || { let mut s = state.borrow_mut(); let voice_channel = s.ui.voice_target.take(); + let was_in_voice = s.audio.is_some(); crate::ui::update::stop_voice(&mut s); + if was_in_voice { + crate::ui::sounds::play(&s, SoundEvent::VoiceLeft); + } s.ui.showing_voice = false; s.ui.ptt_held = false; let net = s.net.clone(); diff --git a/src/ui/bridge.rs b/src/ui/bridge.rs index d0dc29c..a10c972 100644 --- a/src/ui/bridge.rs +++ b/src/ui/bridge.rs @@ -418,6 +418,11 @@ pub fn sync_settings(window: &MainWindow, state: &UiState, own_pubkey: &str) { window.set_voice_bitrate(state.voice_bitrate); window.set_vad_mode(state.vad_mode as i32); window.set_volume(state.output_volume); + window.set_sounds_enabled(state.sounds.enabled); + window.set_sounds_volume(state.sounds.volume); + window.set_sounds_voice_room(state.sounds.voice_room); + window.set_sounds_messages(state.sounds.messages); + window.set_sounds_mentions(state.sounds.mentions); window.set_settings_open(state.settings_open); window.set_input_device_index(state.mic_device as i32); window.set_output_device_index(state.output_device as i32); diff --git a/src/ui/mod.rs b/src/ui/mod.rs index f02cbf0..819e4cc 100644 --- a/src/ui/mod.rs +++ b/src/ui/mod.rs @@ -6,6 +6,7 @@ pub mod i18n; pub mod media; pub mod preferences; pub mod slint; +pub mod sounds; pub mod state; pub mod update; diff --git a/src/ui/preferences.rs b/src/ui/preferences.rs index 372aecf..34e316a 100644 --- a/src/ui/preferences.rs +++ b/src/ui/preferences.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use super::state::{DEFAULT_VOICE_MODE, UiState}; use vnox_client::audio::devices::{self, filter}; +use vnox_client::audio::sounds::SoundSettings; #[derive(Default, Serialize, Deserialize)] #[serde(default)] @@ -22,6 +23,8 @@ struct AudioPreferences { /// stored `mode: 0` for everybody, so an unmarked push-to-talk is not a real choice. mode_chosen: bool, volume: Option, + /// Notification sound switches and volume. + sounds: SoundSettings, } fn path() -> PathBuf { @@ -67,6 +70,10 @@ pub fn load(state: &mut UiState) { state.voice_bitrate = prefs.bitrate.unwrap_or(64.0).clamp(8.0, 128.0); (state.vad_mode, state.vad_mode_chosen) = resolve_mode(&prefs); state.output_volume = prefs.volume.unwrap_or(90.0).clamp(0.0, 100.0); + state.sounds = SoundSettings { + volume: prefs.sounds.volume.clamp(0.0, 100.0), + ..prefs.sounds + }; } pub fn save(state: &UiState) -> anyhow::Result<()> { @@ -83,6 +90,7 @@ pub fn save(state: &UiState) -> anyhow::Result<()> { mode: state.vad_mode, mode_chosen: state.vad_mode_chosen, volume: Some(state.output_volume), + sounds: state.sounds, }; std::fs::write(path, serde_json::to_vec_pretty(&preferences)?)?; Ok(()) diff --git a/src/ui/slint/main.slint b/src/ui/slint/main.slint index 2878e2f..82c3b10 100644 --- a/src/ui/slint/main.slint +++ b/src/ui/slint/main.slint @@ -85,6 +85,13 @@ export component MainWindow inherits Window { in-out property voice-bitrate: 64.0; in-out property vad-mode: 1; in-out property volume: 100.0; + in-out property sounds-enabled: true; + in-out property sounds-volume: 60; + in-out property sounds-voice-room: true; + in-out property sounds-messages: true; + in-out property sounds-mentions: true; + callback sound-settings-changed(bool, float, bool, bool, bool); + callback preview-sound(); in-out property language-choice: "system"; callback language-selected(string); in-out property theme-mode: "dark"; @@ -331,6 +338,13 @@ export component MainWindow inherits Window { voice-bitrate <=> root.voice-bitrate; vad-mode <=> root.vad-mode; volume <=> root.volume; + sounds-enabled <=> root.sounds-enabled; + sounds-volume <=> root.sounds-volume; + sounds-voice-room <=> root.sounds-voice-room; + sounds-messages <=> root.sounds-messages; + sounds-mentions <=> root.sounds-mentions; + sounds-changed => { root.sound-settings-changed(root.sounds-enabled, root.sounds-volume, root.sounds-voice-room, root.sounds-messages, root.sounds-mentions); } + preview-sound => { root.preview-sound(); } active-tab <=> root.active-tab; passphrase-modal-open <=> root.passphrase-modal-open; diff --git a/src/ui/slint/settings.slint b/src/ui/slint/settings.slint index 4701a5e..0ef205d 100644 --- a/src/ui/slint/settings.slint +++ b/src/ui/slint/settings.slint @@ -1,5 +1,6 @@ import { Theme, ActionButton, Symbol } from "theme.slint"; import { AudioSettings } from "settings/audio.slint"; +import { SoundSettings } from "settings/sounds.slint"; import { AppearanceSettings } from "settings/appearance.slint"; import { IdentitySettings } from "settings/identity.slint"; @@ -46,6 +47,13 @@ export component SettingsWindow inherits Rectangle { in-out property voice-bitrate; in-out property vad-mode; in-out property volume; + in-out property sounds-enabled: true; + in-out property sounds-volume: 60; + in-out property sounds-voice-room: true; + in-out property sounds-messages: true; + in-out property sounds-mentions: true; + callback sounds-changed(); + callback preview-sound(); in-out property active-tab: "audio"; in-out property passphrase-modal-open; in-out property passphrase-input; @@ -132,6 +140,15 @@ export component SettingsWindow inherits Rectangle { volume <=> root.volume; audio-changed => { root.audio-changed(); } } + if root.active-tab == "audio" || (root.active-tab != "appearance" && root.active-tab != "identity") : SoundSettings { + sounds-enabled <=> root.sounds-enabled; + sounds-volume <=> root.sounds-volume; + sounds-voice-room <=> root.sounds-voice-room; + sounds-messages <=> root.sounds-messages; + sounds-mentions <=> root.sounds-mentions; + sounds-changed => { root.sounds-changed(); } + preview-sound => { root.preview-sound(); } + } if root.active-tab == "appearance" : AppearanceSettings { language-choice <=> root.language-choice; language-selected(choice) => { root.language-selected(choice); } diff --git a/src/ui/slint/settings/sounds.slint b/src/ui/slint/settings/sounds.slint new file mode 100644 index 0000000..ae74a3e --- /dev/null +++ b/src/ui/slint/settings/sounds.slint @@ -0,0 +1,86 @@ +import { Theme, ActionButton } from "../theme.slint"; +import { AudioSlider } from "../controls.slint"; + +component SoundToggle inherits Rectangle { + in property label; + in property detail; + in-out property checked; + in property enabled: true; + callback toggled(); + height: 46px; + accessible-role: checkbox; + accessible-label: root.label; + accessible-checked: root.checked; + HorizontalLayout { + spacing: 12px; + VerticalLayout { + alignment: center; spacing: 2px; horizontal-stretch: 1; + Text { text: root.label; color: root.enabled ? Theme.text-primary : Theme.text-disabled; font-size: Math.max(Theme.font-size - 2px, 11px); } + Text { text: root.detail; color: Theme.text-muted; font-size: Math.max(Theme.font-size - 4px, 11px); overflow: elide; } + } + ActionButton { + text: root.checked ? "✓" : "○"; width: 34px; height: 30px; quiet: true; + clicked => { if root.enabled { root.checked = !root.checked; root.toggled(); } } + } + } +} + +export component SoundSettings inherits Rectangle { + in-out property sounds-enabled: true; + in-out property sounds-volume: 60; + in-out property sounds-voice-room: true; + in-out property sounds-messages: true; + in-out property sounds-mentions: true; + callback sounds-changed(); + callback preview-sound(); + VerticalLayout { + spacing: 0; + alignment: start; + Rectangle { height: 1px; background: Theme.border; } + Rectangle { height: 26px; } + Text { text: @tr("Sounds"); color: Theme.text-primary; font-size: Theme.font-size + 2px; font-weight: 600; } + Text { text: @tr("Short tones for voice rooms, messages and connection changes. They stay silent while you are deafened."); color: Theme.text-muted; font-size: Math.max(Theme.font-size - 3px, 11px); wrap: word-wrap; } + Rectangle { height: 14px; } + SoundToggle { + label: @tr("Play sounds"); + detail: @tr("Master switch, also covers connection and error tones"); + checked <=> root.sounds-enabled; + toggled => { root.sounds-changed(); } + } + SoundToggle { + label: @tr("Voice room"); + detail: @tr("Joining, leaving, microphone and sound on or off"); + enabled: root.sounds-enabled; + checked <=> root.sounds-voice-room; + toggled => { root.sounds-changed(); } + } + SoundToggle { + label: @tr("Messages"); + detail: @tr("New messages in chats you are not looking at"); + enabled: root.sounds-enabled; + checked <=> root.sounds-messages; + toggled => { root.sounds-changed(); } + } + SoundToggle { + label: @tr("Mentions"); + detail: @tr("Someone mentions you or everyone"); + enabled: root.sounds-enabled; + checked <=> root.sounds-mentions; + toggled => { root.sounds-changed(); } + } + Rectangle { height: 10px; } + HorizontalLayout { + spacing: 20px; + Text { text: @tr("Sound volume"); color: Theme.text-primary; font-size: Theme.font-size - 1px; font-weight: 600; horizontal-stretch: 1; } + Text { text: Math.round(root.sounds-volume) + "%"; color: Theme.text-secondary; font-size: Math.max(Theme.font-size - 2px, 11px); horizontal-alignment: right; } + } + Rectangle { height: 10px; } + AudioSlider { minimum: 0; maximum: 100; value <=> root.sounds-volume; changed(value) => { root.sounds-changed(); } } + Rectangle { height: 12px; } + HorizontalLayout { + ActionButton { text: @tr("Play a test sound"); quiet: true; clicked => { root.preview-sound(); } } + Rectangle { horizontal-stretch: 1; } + } + Rectangle { height: 26px; } + } +} diff --git a/src/ui/sounds.rs b/src/ui/sounds.rs new file mode 100644 index 0000000..758e2fa --- /dev/null +++ b/src/ui/sounds.rs @@ -0,0 +1,217 @@ +//! Maps application events to notification sounds and plays them. +use std::sync::OnceLock; + +use vnox_client::audio::sounds::{self, SoundEvent, SoundPlayer}; +use vnox_client::net::NetEvent; + +use crate::AppState; +use crate::ui::state::{ConnState, UiState, VoiceUiState}; + +fn player() -> &'static SoundPlayer { + static PLAYER: OnceLock = OnceLock::new(); + PLAYER.get_or_init(SoundPlayer::default) +} + +/// Plays `event` if the sound settings and the deafen state allow it. +pub fn play(state: &AppState, event: SoundEvent) { + if let Some(event) = sounds::select(event, &state.ui.sounds, state.ui.deafen) { + player().play( + event, + state.ui.sounds.gain(), + state.audio.as_ref(), + &state.ui.output_device_id, + ); + } +} + +/// Plays the test sound from the settings page, ignoring the category switches but not the +/// master switch or volume. +pub fn preview(state: &AppState) { + let mut settings = state.ui.sounds; + settings.voice_room = true; + if let Some(event) = sounds::select(SoundEvent::PeerJoined, &settings, false) { + player().play( + event, + settings.gain(), + state.audio.as_ref(), + &state.ui.output_device_id, + ); + } +} + +/// True when the person is looking at the text channel `channel_id`. +fn viewing_channel(ui: &UiState, channel_id: &str) -> bool { + !ui.showing_voice + && !ui.browsing_dms + && ui.active_dm_id.is_none() + && ui.active_channel.as_deref() == Some(channel_id) +} + +/// The sound a network event should make, before settings are applied. +pub fn sound_for_event( + ui: &UiState, + own_id: &str, + own_nickname: &str, + event: &NetEvent, +) -> Option { + match event { + NetEvent::Connected { .. } => Some(SoundEvent::Connected), + // Only a connection that was established can be lost; failed attempts stay quiet. + NetEvent::Disconnected { .. } if matches!(ui.conn, ConnState::Connected { .. }) => { + Some(SoundEvent::ConnectionLost) + } + NetEvent::Error { .. } => Some(SoundEvent::Error), + NetEvent::UserJoin { + channel_id, + user_id, + .. + } if user_id != own_id && in_current_voice_room(ui, channel_id) => { + Some(SoundEvent::PeerJoined) + } + NetEvent::UserLeave { + channel_id, + user_id, + } if user_id != own_id && in_current_voice_room(ui, channel_id) => { + Some(SoundEvent::PeerLeft) + } + NetEvent::ChatMessage { + channel_id, + sender_id, + content, + .. + } => sounds::message_event( + sender_id == own_id, + sounds::is_mention(content, own_nickname), + viewing_channel(ui, channel_id), + ), + NetEvent::DmMessage { + dm_id, sender_id, .. + } => sounds::message_event( + sender_id == own_id, + false, + !ui.showing_voice && ui.active_dm_id.as_deref() == Some(dm_id.as_str()), + ), + _ => None, + } +} + +fn in_current_voice_room(ui: &UiState, channel_id: &str) -> bool { + matches!(&ui.voice, VoiceUiState::Active { channel } if channel == channel_id) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn connected_ui() -> UiState { + UiState { + conn: ConnState::Connected { + session_id: "s".into(), + node_name: "n".into(), + }, + active_channel: Some("general".into()), + ..UiState::default() + } + } + + fn chat(channel: &str, sender: &str, content: &str) -> NetEvent { + NetEvent::ChatMessage { + message_id: "m".into(), + channel_id: channel.into(), + sender_id: sender.into(), + nickname: sender.into(), + content: content.into(), + timestamp: 0, + reply_to: None, + } + } + + #[test] + fn messages_sound_only_outside_the_viewed_channel() { + let ui = connected_ui(); + assert_eq!( + sound_for_event(&ui, "me", "me", &chat("general", "bob", "hi")), + None + ); + assert_eq!( + sound_for_event(&ui, "me", "me", &chat("random", "bob", "hi")), + Some(SoundEvent::Message) + ); + assert_eq!( + sound_for_event(&ui, "me", "me", &chat("random", "me", "hi")), + None + ); + } + + #[test] + fn viewing_the_voice_page_does_not_count_as_viewing_the_channel() { + let ui = UiState { + showing_voice: true, + ..connected_ui() + }; + assert_eq!( + sound_for_event(&ui, "me", "me", &chat("general", "bob", "hi")), + Some(SoundEvent::Message) + ); + } + + #[test] + fn mentions_use_the_mention_sound_even_in_the_viewed_channel() { + let ui = connected_ui(); + assert_eq!( + sound_for_event(&ui, "me", "Alice", &chat("general", "bob", "hey @alice")), + Some(SoundEvent::Mention) + ); + } + + #[test] + fn peers_are_announced_only_in_the_voice_room_you_are_in() { + let ui = UiState { + voice: VoiceUiState::Active { + channel: "lounge".into(), + }, + ..connected_ui() + }; + let join = |channel: &str, user: &str| NetEvent::UserJoin { + channel_id: channel.into(), + user_id: user.into(), + nickname: user.into(), + }; + assert_eq!( + sound_for_event(&ui, "me", "me", &join("lounge", "bob")), + Some(SoundEvent::PeerJoined) + ); + assert_eq!( + sound_for_event(&ui, "me", "me", &join("lounge", "me")), + None + ); + assert_eq!( + sound_for_event(&ui, "me", "me", &join("general", "bob")), + None + ); + let leave = NetEvent::UserLeave { + channel_id: "lounge".into(), + user_id: "bob".into(), + }; + assert_eq!( + sound_for_event(&ui, "me", "me", &leave), + Some(SoundEvent::PeerLeft) + ); + } + + #[test] + fn only_an_established_connection_can_be_lost() { + let lost = NetEvent::Disconnected { + reason: "x".into(), + will_retry: true, + }; + assert_eq!( + sound_for_event(&connected_ui(), "me", "me", &lost), + Some(SoundEvent::ConnectionLost) + ); + assert_eq!( + sound_for_event(&UiState::default(), "me", "me", &lost), + None + ); + } +} diff --git a/src/ui/state/default.rs b/src/ui/state/default.rs index 9cdf67f..f2f6f0a 100644 --- a/src/ui/state/default.rs +++ b/src/ui/state/default.rs @@ -43,6 +43,7 @@ impl Default for UiState { output_device: 0, output_device_id: String::new(), output_volume: 90.0, + sounds: Default::default(), device_lists: Default::default(), vad_mode_chosen: false, diff --git a/src/ui/state/types.rs b/src/ui/state/types.rs index 7817458..1c666c9 100644 --- a/src/ui/state/types.rs +++ b/src/ui/state/types.rs @@ -90,6 +90,7 @@ pub struct UiState { /// Persisted output device id; empty is the system default. pub output_device_id: String, pub output_volume: f32, + pub sounds: vnox_client::audio::sounds::SoundSettings, /// Devices behind the picker positions, as of the last enumeration. pub device_lists: vnox_client::audio::DeviceLists, /// True once the person picked a voice input mode themselves. diff --git a/src/ui/update.rs b/src/ui/update.rs index 4ae73b4..2c64c44 100644 --- a/src/ui/update.rs +++ b/src/ui/update.rs @@ -9,6 +9,14 @@ use crate::AppState; pub fn process_net_events(state: &mut AppState) { while let Some(event) = state.net.try_recv() { + if let Some(sound) = crate::ui::sounds::sound_for_event( + &state.ui, + &state.identity.pubkey_hex, + &state.identity.nickname, + &event, + ) { + crate::ui::sounds::play(state, sound); + } match event { NetEvent::VoiceRevoked { channel_id } => { if state.voice_channel.as_deref() == Some(channel_id.as_str()) @@ -895,6 +903,8 @@ pub fn start_voice(state: &mut AppState, channel_id: &str) { match vnox_client::audio::start(cfg.clone()) { Ok(pipeline) => { tracing::info!("voice active on channel {channel_id}"); + let newly_joined = !matches!(&state.ui.voice, + VoiceUiState::Active { channel } if channel == channel_id); state.audio = Some(pipeline); state.voice_channel = Some(channel_id.to_string()); state.last_pipeline_config = Some(cfg); @@ -905,6 +915,9 @@ pub fn start_voice(state: &mut AppState, channel_id: &str) { state.ui.voice_joined_at = Some(std::time::Instant::now()); } state.ui.status_msg = None; + if newly_joined { + crate::ui::sounds::play(state, vnox_client::audio::sounds::SoundEvent::VoiceJoined); + } } Err(e) => { tracing::warn!("audio start failed: {e}"); @@ -983,6 +996,7 @@ fn report_audio_issues(state: &mut AppState) { ), }; state.ui.status_msg = Some(message); + crate::ui::sounds::play(state, vnox_client::audio::sounds::SoundEvent::Error); } if dead { let reason = state diff --git a/translations/en/LC_MESSAGES/vnox-client.po b/translations/en/LC_MESSAGES/vnox-client.po index 08cef80..8e70ef5 100644 --- a/translations/en/LC_MESSAGES/vnox-client.po +++ b/translations/en/LC_MESSAGES/vnox-client.po @@ -271,6 +271,9 @@ msgstr "Join" msgid "Joining voice…" msgstr "Joining voice…" +msgid "Joining, leaving, microphone and sound on or off" +msgstr "Joining, leaving, microphone and sound on or off" + msgid "Keep your keys in a safe place to restore your profile on another device." msgstr "Keep your keys in a safe place to restore your profile on another device." @@ -304,15 +307,24 @@ msgstr "Make Vnox yours." msgid "Make yourself heard, your way." msgstr "Make yourself heard, your way." +msgid "Master switch, also covers connection and error tones" +msgstr "Master switch, also covers connection and error tones" + msgid "Members" msgstr "Members" msgid "Members — {}" msgstr "Members — {}" +msgid "Mentions" +msgstr "Mentions" + msgid "Message" msgstr "Message" +msgid "Messages" +msgstr "Messages" + msgid "Microphone" msgstr "Microphone" @@ -331,6 +343,9 @@ msgstr "Mute microphone" msgid "New deliveries stop. Existing messages stay in the channel." msgstr "New deliveries stop. Existing messages stay in the channel." +msgid "New messages in chats you are not looking at" +msgstr "New messages in chats you are not looking at" + msgid "New passphrase · at least 8 characters" msgstr "New passphrase · at least 8 characters" @@ -403,6 +418,12 @@ msgstr "Photo" msgid "Photo crop area. Drag the image to adjust the crop." msgstr "Photo crop area. Drag the image to adjust the crop." +msgid "Play a test sound" +msgstr "Play a test sound" + +msgid "Play sounds" +msgstr "Play sounds" + msgid "Playback volume for voice rooms." msgstr "Playback volume for voice rooms." @@ -523,6 +544,18 @@ msgstr "Settings for this device" msgid "Share audio" msgstr "Share audio" +msgid "Short tones for voice rooms, messages and connection changes. They stay silent while you are deafened." +msgstr "Short tones for voice rooms, messages and connection changes. They stay silent while you are deafened." + +msgid "Someone mentions you or everyone" +msgstr "Someone mentions you or everyone" + +msgid "Sound volume" +msgstr "Sound volume" + +msgid "Sounds" +msgstr "Sounds" + msgid "Speaking" msgstr "Speaking" @@ -631,6 +664,9 @@ msgstr "Voice input" msgid "Voice quality" msgstr "Voice quality" +msgid "Voice room" +msgstr "Voice room" + msgid "Voice rooms" msgstr "Voice rooms" diff --git a/translations/ru.json b/translations/ru.json index 27afbee..0e05c7c 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -203,6 +203,7 @@ "Could not remove photo: {error}": "Не удалось удалить фото: {error}", "Language": "Язык", "System default": "Как в системе", + "Default audio device": "По умолчанию (системное)", "Choose your language. Changes apply immediately.": "Выбери язык. Изменения применяются сразу.", "{n} participant": [ "{n} участник", @@ -231,7 +232,6 @@ "Idle": "Нет на месте", "Do not disturb": "Не беспокоить", "Default": "По умолчанию", - "System default": "По умолчанию (системное)", "Reply to {}": "Ответ {}", "Members — {}": "Участники — {}", "Zoom {}%": "Масштаб {}%", @@ -366,5 +366,17 @@ "Forget this server": "Забыть этот сервер", "Trust and connect": "Доверять и подключиться", "Could not save the trusted fingerprint": "Не удалось сохранить доверенный отпечаток", - "Server forgotten. Connect again to review its fingerprint.": "Сервер забыт. Подключись снова, чтобы проверить его отпечаток." + "Server forgotten. Connect again to review its fingerprint.": "Сервер забыт. Подключись снова, чтобы проверить его отпечаток.", + "Sounds": "Звуки", + "Short tones for voice rooms, messages and connection changes. They stay silent while you are deafened.": "Короткие сигналы для голосовых комнат, сообщений и смены состояния связи. Пока звук выключен, они молчат.", + "Play sounds": "Воспроизводить звуки", + "Master switch, also covers connection and error tones": "Общий переключатель, включая сигналы связи и ошибок", + "Voice room": "Голосовая комната", + "Joining, leaving, microphone and sound on or off": "Вход, выход, микрофон и включение или выключение звука", + "Messages": "Сообщения", + "New messages in chats you are not looking at": "Новые сообщения в чатах, которые вы сейчас не смотрите", + "Mentions": "Упоминания", + "Someone mentions you or everyone": "Вас или всех упомянули", + "Sound volume": "Громкость звуков", + "Play a test sound": "Проверить звук" } diff --git a/translations/ru/LC_MESSAGES/vnox-client.po b/translations/ru/LC_MESSAGES/vnox-client.po index 77efa4e..352f52c 100644 --- a/translations/ru/LC_MESSAGES/vnox-client.po +++ b/translations/ru/LC_MESSAGES/vnox-client.po @@ -272,6 +272,9 @@ msgstr "Войти" msgid "Joining voice…" msgstr "Подключение к голосовой комнате…" +msgid "Joining, leaving, microphone and sound on or off" +msgstr "Вход, выход, микрофон и включение или выключение звука" + msgid "Keep your keys in a safe place to restore your profile on another device." msgstr "Сохрани ключи в безопасном месте, чтобы восстановить профиль на другом устройстве." @@ -305,15 +308,24 @@ msgstr "Сделай Vnox своим." msgid "Make yourself heard, your way." msgstr "Голос должен звучать так, как тебе удобно." +msgid "Master switch, also covers connection and error tones" +msgstr "Общий переключатель, включая сигналы связи и ошибок" + msgid "Members" msgstr "Участники" msgid "Members — {}" msgstr "Участники — {}" +msgid "Mentions" +msgstr "Упоминания" + msgid "Message" msgstr "Написать" +msgid "Messages" +msgstr "Сообщения" + msgid "Microphone" msgstr "Микрофон" @@ -332,6 +344,9 @@ msgstr "Выключить микрофон" msgid "New deliveries stop. Existing messages stay in the channel." msgstr "Новые доставки прекратятся. Отправленные сообщения останутся в канале." +msgid "New messages in chats you are not looking at" +msgstr "Новые сообщения в чатах, которые вы сейчас не смотрите" + msgid "New passphrase · at least 8 characters" msgstr "Новый пароль · минимум 8 символов" @@ -404,6 +419,12 @@ msgstr "Фото" msgid "Photo crop area. Drag the image to adjust the crop." msgstr "Область фото. Перетаскивай изображение, чтобы изменить кадрирование." +msgid "Play a test sound" +msgstr "Проверить звук" + +msgid "Play sounds" +msgstr "Воспроизводить звуки" + msgid "Playback volume for voice rooms." msgstr "Общий уровень воспроизведения голосовых комнат." @@ -524,6 +545,18 @@ msgstr "Настройки этого устройства" msgid "Share audio" msgstr "Звук демонстрации" +msgid "Short tones for voice rooms, messages and connection changes. They stay silent while you are deafened." +msgstr "Короткие сигналы для голосовых комнат, сообщений и смены состояния связи. Пока звук выключен, они молчат." + +msgid "Someone mentions you or everyone" +msgstr "Вас или всех упомянули" + +msgid "Sound volume" +msgstr "Громкость звуков" + +msgid "Sounds" +msgstr "Звуки" + msgid "Speaking" msgstr "Говорит" @@ -543,7 +576,7 @@ msgid "Stop watching" msgstr "Закрыть просмотр" msgid "System default" -msgstr "По умолчанию (системное)" +msgstr "Как в системе" msgid "System output" msgstr "Системный звук" @@ -632,6 +665,9 @@ msgstr "Передача голоса" msgid "Voice quality" msgstr "Качество голоса" +msgid "Voice room" +msgstr "Голосовая комната" + msgid "Voice rooms" msgstr "Голосовые комнаты" From 14543f981f9a0e43c7f323a6498192772626fe04 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Sun, 25 Oct 2026 10:02:17 +0100 Subject: [PATCH 6/6] docs: record causes and fixes for the audio defects --- docs/maintenance/ux-defects-2026-10-24.md | 35 ++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/maintenance/ux-defects-2026-10-24.md b/docs/maintenance/ux-defects-2026-10-24.md index 7bf5420..8506bd8 100644 --- a/docs/maintenance/ux-defects-2026-10-24.md +++ b/docs/maintenance/ux-defects-2026-10-24.md @@ -14,14 +14,14 @@ | 6 | Шапка сервера | Кнопка «питание» рядом с названием сервера непонятна | A | открыто | | 7 | Сообщения | В панели при наведении («Ответить, Править, Удалить») странная иконка слева | A | открыто | | 8 | Эмодзи | Нет выбора эмодзи для текста сообщения и для реакций на сообщения | A | открыто | -| 9 | Звук | В голосовой комнате нет звуков: вход и выход участников, сообщения, упоминание или пинг, mute и unmute, подключение | B | открыто | -| 10a | Голос | По умолчанию стоит режим рации (PTT), выглядит странно и неудобно; нужен обычный режим по активации голоса | B | открыто | +| 9 | Звук | В голосовой комнате нет звуков: вход и выход участников, сообщения, упоминание или пинг, mute и unmute, подключение | B исправлено: звуки событий синтезируются программно, настройки в разделе «Звуки», диагностика ошибок аудио (см. ниже) | +| 10a | Голос | По умолчанию стоит режим рации (PTT), выглядит странно и неудобно; нужен обычный режим по активации голоса | B исправлено: по умолчанию голосовая активность с автоподбором порога | | 10b | Голосовая комната | Страница слишком серая, текстовые кнопки «Выключить микрофон», «Выключить звук», «Покинуть комнату» неудобны; нужны иконочные кнопки | A | открыто | -| 10c | Голосовая комната | Верхняя строка «Подключение к голосовой комнате…» остаётся, хотя внизу «Голос подключён» | B | открыто | +| 10c | Голосовая комната | Верхняя строка «Подключение к голосовой комнате…» остаётся, хотя внизу «Голос подключён» | B исправлено: единый статус голоса (см. ниже) | | 10d | Аккаунт | Можно зайти на сервер без аккаунта; регистрация или вход в Central должны быть обязательными | C | открыто | | 10e | Аватары | Две буквы вместо аватара; при первой регистрации нужен случайный сгенерированный аватар | C | открыто | | 10f | Отзывчивость | Приложение реагирует медленно; нажатие срабатывает не с одного клика, а со второго | A | открыто | -| 11a | Настройки звука | В списках устройств вывода и микрофона мусорные ALSA-пункты (Discard all samples, Rate Converter Plugin, JACK, Open Sound System) | B | открыто | +| 11a | Настройки звука | В списках устройств вывода и микрофона мусорные ALSA-пункты (Discard all samples, Rate Converter Plugin, JACK, Open Sound System) | B исправлено: фильтр устройств, выбор хранится по идентификатору | | 11b | Настройки | Те же двойные клики в настройках | A | открыто | | 12a | Темы | Тема работает странно (чёрная тема: части интерфейса остаются серыми, фон не везде одинаковый) | A | открыто | | 12b | Аватар | Аватар в панели пользователя обновился, в карточке участника голосовой комнаты остались две буквы | A, C | открыто | @@ -33,3 +33,30 @@ - Отзывчивость: сначала измерить причину (двойной клик, ожидание таймера, фокус), потом чинить, не угадывать. - Любое изменение протокола сначала в документации (правило репозитория), затем в коде. - Никаких длинных тире в пользовательских текстах. + +## Поток B: причины и исправления + +### Нет звука в голосовой комнате (пункты 9 и 10a) + +Транспорт исправен: `vnox-e2e-voice` против локального `vnox-serverd` проходит (TLS 1.3, UDP-реле, обмен пакетами в обе стороны). Сбои были на стороне клиента: + +1. Режим по умолчанию был рацией (PTT), а кнопка «Hold here to talk» нужна для каждой фразы. Без неё исходящий голос не отправлялся совсем, то есть собеседник «не слышал». Теперь по умолчанию голосовая активность. +2. Порог VAD был примерно -26 dBFS (RMS 0,049 при настройке 40), обычная речь с микрофона без усиления его почти не пересекала. Теперь потолок около -34 dBFS, а порог автоматически опускается чуть выше измеренного шума комнаты (минимум около -48 dBFS), с задержкой закрытия 300 мс. +3. Ошибки открытия потоков захвата и воспроизведения (устройство занято или не даёт f32, пропавшее устройство, отказ `play`) только писались в лог из отдельного потока, а интерфейс показывал «Голос подключён». Теперь потоки сообщают об ошибках в общий приёмник, в строке состояния появляется «Ошибка микрофона» или «Ошибка вывода звука», при отказе обеих половин комната помечается ошибкой. Микрофон, отдающий только цифровую тишину, тоже сообщается. Раз в 10 секунд в лог пишутся счётчики (захвачено, отправлено, получено, декодировано, пик входа), а отклонённые UDP-дейтаграммы (чужой адрес источника, ошибка аутентификации) больше не отбрасываются молча. +4. Выбор формата: принимаются только конфигурации f32 (так построены потоки), для устройств без f32 выдаётся понятная ошибка вместо отказа при открытии потока. + +Не подтверждено без доступа к продакшену: что именно слышал пользователь на `central.loki-code.dev`. Если после этих исправлений тишина останется, смотреть строку «voice audio stats» и предупреждения «voice datagram ... ignored» в логе: по ним видно, остановился ли сигнал на микрофоне, в сети или на выходе. Подозрение, которое проверяется только на сервере: адрес `voice.bind` на продакшене должен быть достижим клиентом, иначе пакеты придут с другого адреса и будут проигнорированы. + +### Статус подключения (10c) + +Причина: при клике на голосовую комнату ставилось свободное сообщение «Joining voice channel…», которое сбрасывалось только при запуске аудио. Повторный клик по комнате, в которой клиент уже находится, сервер не подтверждает, поэтому сообщение оставалось навсегда. Кроме того, страница голоса показывала «Joining voice…» при любом пустом списке участников. Теперь `UiState::voice_status` выводится из состояния соединения и голоса, ожидающий вход хранится отдельно (`pending_voice_join`), боковая панель, страница голоса и статус читают один источник. Повторный клик только показывает комнату. При переходе из комнаты A в B живая сессия A остаётся «подключено» до ответа сервера. Конвейер захвата останавливается всякий раз, когда он не принадлежит текущей голосовой цели (например после удаления комнаты). + +### Устройства (11a) + +Фильтр в `src/audio/devices/filter.rs`: остаются «По умолчанию (системное)», PipeWire и PulseAudio и по одной записи на аппаратную точку (`sysdefault`, затем `plughw`, `hw`, `hdmi`), без технических хвостов в названиях, отсортировано. Выбор хранится по идентификатору устройства; индексы старых версий переносятся по имени, пропавшее устройство откатывается на системное. + +### Звуки (пункт 9) + +События: вход и выход из комнаты, вход и выход участника, новое сообщение вне просматриваемого чата, упоминание (`@ник`, `@everyone`, `@here`), выключение и включение микрофона и звука, подключение, потеря связи, ошибка. Источник: программный синтез (`src/audio/sounds/synth.rs`), сторонних файлов нет, см. `docs/THIRD_PARTY.md`. Настройки: общий переключатель, громкость, переключатели «Голосовая комната», «Сообщения», «Упоминания». Сигналы связи и ошибок зависят только от общего переключателя. При выключенном звуке (deafen) звучит только сигнал его включения. + +Ограничение: Slint не отдаёт состояние фокуса окна ОС, поэтому «новое сообщение» определяется по тому, какой чат открыт, а не по фокусу окна.