fix #14740: record detected calls from the output device

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
hauke
2026-11-06 10:49:02 +01:00
co-authored by Claude Opus 5.5
parent 1cb1033cbb
commit 5dfe1d9357
12 changed files with 397 additions and 346 deletions
-218
View File
@@ -1,218 +0,0 @@
//! Output capture that falls back from per-process to device loopback.
//!
//! Application loopback (`OutputTarget::Process`) needs Windows 10 2004+ and
//! a live process; if it cannot be opened, the call is still worth recording
//! from the default output. [`crate::Recorder`] keeps a failed track running
//! as silence instead of retrying, so the retry happens here, inside
//! `start`.
use std::sync::{Arc, Mutex, PoisonError};
use recorder_core::capture::{
CaptureError, CaptureSource, DataCallback, OutputTarget, StreamFormat,
};
/// Creates the platform source for one output target.
pub(crate) type Factory = fn(OutputTarget) -> Box<dyn CaptureSource>;
/// Tries `target`, and if that is a `Process` target that fails to start,
/// `DefaultRender`. `stop` and `failure` act on whichever source started.
pub(crate) struct FallbackSource {
target: OutputTarget,
factory: Factory,
active: Option<Box<dyn CaptureSource>>,
}
impl FallbackSource {
pub(crate) fn new(target: OutputTarget, factory: Factory) -> Self {
Self {
target,
factory,
active: None,
}
}
}
impl CaptureSource for FallbackSource {
fn start(&mut self, on_data: DataCallback) -> Result<StreamFormat, CaptureError> {
self.stop();
// A failed start consumes the callback it was given, so each attempt
// gets a forwarder to the one shared callback.
let on_data = Arc::new(Mutex::new(on_data));
let forward = |on_data: &Arc<Mutex<DataCallback>>| -> DataCallback {
let on_data = Arc::clone(on_data);
Box::new(move |samples| {
(on_data.lock().unwrap_or_else(PoisonError::into_inner))(samples)
})
};
let mut source = (self.factory)(self.target.clone());
let format = match source.start(forward(&on_data)) {
Ok(format) => {
log::info!("output target {:?} started", self.target);
format
}
Err(err) => {
let OutputTarget::Process { root_pid } = self.target else {
return Err(err);
};
log::warn!(
"process loopback for pid {root_pid} failed ({err}), \
falling back to DefaultRender"
);
source = (self.factory)(OutputTarget::DefaultRender);
source.start(forward(&on_data))?
}
};
self.active = Some(source);
Ok(format)
}
fn stop(&mut self) {
if let Some(mut source) = self.active.take() {
source.stop();
}
}
fn failure(&self) -> Option<String> {
self.active.as_ref().and_then(|source| source.failure())
}
}
#[cfg(test)]
mod tests {
use super::*;
const FORMAT: StreamFormat = StreamFormat {
rate: 48_000,
channels: 2,
};
/// Per-test record of what the fake sources saw. Tests run in parallel,
/// so each test gets its own log via a thread-local.
#[derive(Default)]
struct Log {
started: Vec<OutputTarget>,
stopped: Vec<OutputTarget>,
delivered: Vec<f32>,
}
thread_local! {
static LOG: Arc<Mutex<Log>> = Arc::new(Mutex::new(Log::default()));
}
fn log() -> Arc<Mutex<Log>> {
LOG.with(Arc::clone)
}
/// Fails to start for `Process` targets whose pid is 0 and for
/// `Device("broken")`; otherwise delivers one sample (the call count)
/// synchronously from `start` and reports a failure for
/// `Device("dying")`.
struct FakeOutput {
target: OutputTarget,
log: Arc<Mutex<Log>>,
started: bool,
}
fn fake(target: OutputTarget) -> Box<dyn CaptureSource> {
Box::new(FakeOutput {
target,
log: log(),
started: false,
})
}
impl CaptureSource for FakeOutput {
fn start(&mut self, mut on_data: DataCallback) -> Result<StreamFormat, CaptureError> {
self.log.lock().unwrap().started.push(self.target.clone());
match &self.target {
OutputTarget::Process { root_pid: 0 } => {
return Err(CaptureError::Backend("no such process".into()))
}
OutputTarget::Device(id) if id == "broken" => {
return Err(CaptureError::DeviceNotFound(id.clone()))
}
_ => {}
}
self.started = true;
on_data(&[1.0]);
Ok(FORMAT)
}
fn stop(&mut self) {
if std::mem::take(&mut self.started) {
self.log.lock().unwrap().stopped.push(self.target.clone());
}
}
fn failure(&self) -> Option<String> {
(self.started && self.target == OutputTarget::Device("dying".into()))
.then(|| "device lost".to_string())
}
}
fn collecting_callback() -> DataCallback {
let log = log();
Box::new(move |samples| log.lock().unwrap().delivered.extend_from_slice(samples))
}
#[test]
fn a_working_process_target_is_used_as_is() {
let mut source = FallbackSource::new(OutputTarget::Process { root_pid: 42 }, fake);
assert_eq!(source.start(collecting_callback()).unwrap(), FORMAT);
source.stop();
let log = log();
let log = log.lock().unwrap();
assert_eq!(log.started, vec![OutputTarget::Process { root_pid: 42 }]);
assert_eq!(log.stopped, vec![OutputTarget::Process { root_pid: 42 }]);
assert_eq!(log.delivered, vec![1.0]);
}
#[test]
fn a_failing_process_target_falls_back_to_default_render() {
let mut source = FallbackSource::new(OutputTarget::Process { root_pid: 0 }, fake);
assert_eq!(source.start(collecting_callback()).unwrap(), FORMAT);
assert_eq!(source.failure(), None);
source.stop();
let log = log();
let log = log.lock().unwrap();
assert_eq!(
log.started,
vec![
OutputTarget::Process { root_pid: 0 },
OutputTarget::DefaultRender
]
);
assert_eq!(log.stopped, vec![OutputTarget::DefaultRender]);
// The callback survives the failed first attempt.
assert_eq!(log.delivered, vec![1.0]);
}
#[test]
fn other_targets_do_not_fall_back() {
let mut source = FallbackSource::new(OutputTarget::Device("broken".into()), fake);
assert!(matches!(
source.start(collecting_callback()),
Err(CaptureError::DeviceNotFound(_))
));
assert_eq!(
log().lock().unwrap().started,
vec![OutputTarget::Device("broken".into())]
);
}
#[test]
fn failure_and_stop_act_on_the_started_source() {
let mut source = FallbackSource::new(OutputTarget::Device("dying".into()), fake);
assert_eq!(source.failure(), None);
source.start(collecting_callback()).unwrap();
assert_eq!(source.failure().as_deref(), Some("device lost"));
source.stop();
assert_eq!(source.failure(), None);
source.stop();
assert_eq!(log().lock().unwrap().stopped.len(), 1);
}
}
+14 -10
View File
@@ -1,17 +1,20 @@
//! Platform audio capture for the Gesprächsrekorder call recorder. //! Platform audio capture for the Gesprächsrekorder call recorder.
//! //!
//! Implements [`recorder_core::capture::CaptureSource`] for the microphone and //! Implements [`recorder_core::capture::CaptureSource`] for the microphone and
//! for the call app's audio output: WASAPI on Windows, PulseAudio on Linux. //! for loopback of an output device: WASAPI on Windows, PulseAudio on Linux.
//! Also lists the processes holding the microphone ([`mic_sessions`]), which //! Also lists the processes holding the microphone ([`mic_sessions`]), which
//! drives call detection on Windows. The platform-neutral trait, types, the //! drives call detection on Windows, and the processes playing on each output
//! two-track recorder and the call detector live in `recorder-core`. //! device ([`render_sessions`]), which picks the device a detected call is
//! recorded from ([`render_endpoint_for`]). The platform-neutral trait,
//! types, the two-track recorder and the call detector live in
//! `recorder-core`.
mod capture_thread; mod capture_thread;
mod fallback;
mod process_tree; mod process_tree;
mod render_sessions;
mod stats; mod stats;
use recorder_core::capture::{CaptureSource, OutputTarget}; pub use render_sessions::{describe_render_sessions, render_endpoint_for, RenderSession};
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
mod linux { mod linux {
@@ -28,7 +31,7 @@ mod windows {
#[cfg(windows)] #[cfg(windows)]
pub use windows::capture::{list_devices, mic_source, output_source}; pub use windows::capture::{list_devices, mic_source, output_source};
#[cfg(windows)] #[cfg(windows)]
pub use windows::sessions::mic_sessions; pub use windows::sessions::{mic_sessions, render_sessions};
/// Processes currently holding a microphone. Call detection is Windows-only; /// Processes currently holding a microphone. Call detection is Windows-only;
/// Linux reports none. /// Linux reports none.
@@ -37,8 +40,9 @@ pub fn mic_sessions() -> Vec<recorder_core::detect::MicSession> {
Vec::new() Vec::new()
} }
/// Output capture for `target`; if `target` is a process whose loopback /// Audio sessions on the output devices. Only needed for detected calls,
/// cannot be opened, records the default output device instead. /// which are Windows-only; Linux reports none.
pub fn output_source_with_fallback(target: OutputTarget) -> Box<dyn CaptureSource> { #[cfg(target_os = "linux")]
Box::new(fallback::FallbackSource::new(target, output_source)) pub fn render_sessions() -> Vec<RenderSession> {
Vec::new()
} }
+3 -4
View File
@@ -2,9 +2,8 @@
//! //!
//! Streams are opened directly as 16 kHz mono `f32`, so the server does the //! Streams are opened directly as 16 kHz mono `f32`, so the server does the
//! resampling and downmix and the recorder's conditioner passes them through. //! resampling and downmix and the recorder's conditioner passes them through.
//! The call output is recorded from the default sink's monitor source; //! The call output is recorded from the default sink's monitor source, or
//! per-process capture does not exist here, so `OutputTarget::Process` falls //! from the source named by `OutputTarget::Device`.
//! back to that monitor as well.
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant; use std::time::Instant;
@@ -49,7 +48,7 @@ pub fn mic_source(device: Option<&str>) -> Box<dyn CaptureSource> {
pub fn output_source(target: OutputTarget) -> Box<dyn CaptureSource> { pub fn output_source(target: OutputTarget) -> Box<dyn CaptureSource> {
let device = match target { let device = match target {
OutputTarget::Process { .. } | OutputTarget::DefaultRender => DEFAULT_MONITOR.to_owned(), OutputTarget::DefaultRender => DEFAULT_MONITOR.to_owned(),
OutputTarget::Device(name) => name, OutputTarget::Device(name) => name,
}; };
Box::new(PulseSource { Box::new(PulseSource {
+7 -6
View File
@@ -1,9 +1,10 @@
//! Finding the root of a multi-process app (Chrome, Edge, Teams). //! Finding the root of a multi-process app (Chrome, Edge, Teams).
//! //!
//! Browsers and Teams open the microphone from a child process, but the //! Browsers and Teams open the microphone from a child process, but the
//! call's audio is played by a sibling. Application loopback records a //! call's audio is played by a sibling. To tie the mic session and the
//! process *tree*, so the recorder needs the top-most process of the app: //! render session to the same call, both are identified by the top-most
//! walk up the parent chain while the parent runs the same executable. //! process of the app: walk up the parent chain while the parent runs the
//! same executable.
//! //!
//! Some apps capture the mic in a helper with a different exe: new Teams //! Some apps capture the mic in a helper with a different exe: new Teams
//! (since Jan 2026) in `ms-teams_modulehost.exe`, earlier in //! (since Jan 2026) in `ms-teams_modulehost.exe`, earlier in
@@ -35,12 +36,12 @@ fn is_one_of(exe: &str, names: &[&str]) -> bool {
names.iter().any(|name| exe.eq_ignore_ascii_case(name)) names.iter().any(|name| exe.eq_ignore_ascii_case(name))
} }
/// The app a mic session belongs to: `(exe, root_pid)`. /// The app an audio session (mic or render) belongs to: `(exe, root_pid)`.
/// ///
/// Normally that is the session process's own `exe` and its same-exe /// Normally that is the session process's own `exe` and its same-exe
/// [`root_pid`]. A Teams helper (`exe` in [`TEAMS_HELPERS`]) is attributed to /// [`root_pid`]. A Teams helper (`exe` in [`TEAMS_HELPERS`]) is attributed to
/// the nearest Teams ancestor instead, reached through helpers only, so /// the nearest Teams ancestor instead, reached through helpers only, so the
/// process loopback of that ancestor's tree covers the helper too. The walk /// helper's sessions count as that Teams process's. The walk
/// trusts a parent only if it was created strictly before its child /// trusts a parent only if it was created strictly before its child
/// (`created` returns a process's creation time); otherwise the parent pid /// (`created` returns a process's creation time); otherwise the parent pid
/// was reused by an unrelated process, or the time is unknown, and the walk /// was reused by an unrelated process, or the time is unknown, and the walk
+139
View File
@@ -0,0 +1,139 @@
//! Which render endpoint a detected call plays on.
//!
//! A detected call's far end is recorded by loopback of an output device.
//! The device is the one on which the call app has an active audio session;
//! a session belongs to the call if its process is attributed to the call's
//! exe (Teams helpers count as Teams, see [`crate::process_tree`]) or sits in
//! the call's process tree. Windows enumerates the sessions; the choice is a
//! pure function over that list so it can be tested anywhere.
/// One audio session on a render endpoint (an app playing, or able to play,
/// on an output device).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RenderSession {
/// The session process's exe, after Teams-helper attribution.
pub exe: String,
pub pid: u32,
/// Top of the process's same-exe parent chain (or its Teams root).
pub root_pid: u32,
/// Whether a stream of the session is running.
pub active: bool,
pub device_id: String,
pub device_name: String,
}
/// The first active session (in enumeration order) that belongs to the call
/// app `exe` (case-insensitive) or to its process tree `root_pid`; its
/// device is where the call's audio plays. `None` if the call app has no
/// active render session.
pub fn render_endpoint_for<'a>(
sessions: &'a [RenderSession],
exe: &str,
root_pid: u32,
) -> Option<&'a RenderSession> {
sessions
.iter()
.find(|s| s.active && (s.exe.eq_ignore_ascii_case(exe) || s.root_pid == root_pid))
}
/// "exe pid N root N on device" per active session, in enumeration order;
/// for seeing in the log which process really plays a call's audio.
pub fn describe_render_sessions(sessions: &[RenderSession]) -> String {
sessions
.iter()
.filter(|s| s.active)
.map(|s| {
format!(
"{} pid {} root {} on {}",
s.exe, s.pid, s.root_pid, s.device_name
)
})
.collect::<Vec<_>>()
.join(", ")
}
#[cfg(test)]
mod tests {
use super::*;
fn session(exe: &str, pid: u32, root_pid: u32, active: bool, device: &str) -> RenderSession {
RenderSession {
exe: exe.to_string(),
pid,
root_pid,
active,
device_id: format!("{{id-{device}}}"),
device_name: device.to_string(),
}
}
fn chosen<'a>(sessions: &'a [RenderSession], exe: &str, root_pid: u32) -> Option<&'a str> {
render_endpoint_for(sessions, exe, root_pid).map(|s| s.device_id.as_str())
}
#[test]
fn a_session_of_the_call_exe_chooses_its_device() {
let sessions = [
session("spotify.exe", 50, 50, true, "Lautsprecher"),
session("MS-Teams.exe", 300, 120, true, "Headset"),
];
assert_eq!(chosen(&sessions, "ms-teams.exe", 100), Some("{id-Headset}"));
}
#[test]
fn a_session_in_the_call_process_tree_chooses_its_device() {
let sessions = [
session("spotify.exe", 50, 50, true, "Lautsprecher"),
session("audiohost.exe", 300, 100, true, "Headset"),
];
assert_eq!(chosen(&sessions, "ms-teams.exe", 100), Some("{id-Headset}"));
}
#[test]
fn inactive_sessions_are_ignored() {
let sessions = [
session("ms-teams.exe", 100, 100, false, "Lautsprecher"),
session("ms-teams.exe", 200, 100, true, "Headset"),
];
assert_eq!(chosen(&sessions, "ms-teams.exe", 100), Some("{id-Headset}"));
assert_eq!(chosen(&sessions[..1], "ms-teams.exe", 100), None);
}
#[test]
fn no_session_of_the_call_app_chooses_nothing() {
let sessions = [
session("spotify.exe", 50, 50, true, "Lautsprecher"),
session("chrome.exe", 70, 60, true, "Headset"),
];
assert_eq!(chosen(&sessions, "ms-teams.exe", 100), None);
assert_eq!(chosen(&[], "ms-teams.exe", 100), None);
}
#[test]
fn the_first_matching_endpoint_wins() {
let sessions = [
session("ms-teams.exe", 100, 100, true, "Headset"),
session("ms-teams.exe", 100, 100, true, "Lautsprecher"),
];
assert_eq!(chosen(&sessions, "ms-teams.exe", 100), Some("{id-Headset}"));
let reversed = [sessions[1].clone(), sessions[0].clone()];
assert_eq!(
chosen(&reversed, "ms-teams.exe", 100),
Some("{id-Lautsprecher}")
);
}
#[test]
fn describe_lists_active_sessions_in_order() {
let sessions = [
session("ms-teams.exe", 300, 100, true, "Headset"),
session("spotify.exe", 50, 50, false, "Lautsprecher"),
session("chrome.exe", 70, 60, true, "Lautsprecher"),
];
assert_eq!(
describe_render_sessions(&sessions),
"ms-teams.exe pid 300 root 100 on Headset, chrome.exe pid 70 root 60 on Lautsprecher"
);
assert_eq!(describe_render_sessions(&[]), "");
}
}
+14 -46
View File
@@ -2,10 +2,9 @@
//! //!
//! - Microphone: the default *communications* capture endpoint, or the //! - Microphone: the default *communications* capture endpoint, or the
//! endpoint with the given device id. //! endpoint with the given device id.
//! - Output, `Process`: application loopback of the call app's process tree //! - Output: loopback of the default communications render endpoint
//! (Windows 10 2004+), so only the call's audio is recorded. //! (`DefaultRender`) or of the render endpoint with the given id
//! - Output, `DefaultRender` / `Device`: loopback of the default //! (`Device`).
//! communications render endpoint / of the given render endpoint.
//! //!
//! All streams are delivered as interleaved `f32`. When the endpoint goes //! All streams are delivered as interleaved `f32`. When the endpoint goes
//! away mid-recording (`AUDCLNT_E_DEVICE_INVALIDATED`, e.g. a headset was //! away mid-recording (`AUDCLNT_E_DEVICE_INVALIDATED`, e.g. a headset was
@@ -28,15 +27,8 @@ use wasapi::{
use crate::capture_thread::{CaptureThread, Ready}; use crate::capture_thread::{CaptureThread, Ready};
use crate::stats::CaptureStats; use crate::stats::CaptureStats;
use super::sessions;
/// `AUDCLNT_E_DEVICE_INVALIDATED`: the endpoint was removed or disabled. /// `AUDCLNT_E_DEVICE_INVALIDATED`: the endpoint was removed or disabled.
const AUDCLNT_E_DEVICE_INVALIDATED: i32 = 0x8889_0004_u32 as i32; const AUDCLNT_E_DEVICE_INVALIDATED: i32 = 0x8889_0004_u32 as i32;
/// Format requested for application loopback, which has no mix format.
const PROCESS_LOOPBACK_FORMAT: StreamFormat = StreamFormat {
rate: 48_000,
channels: 2,
};
/// Shared-mode buffer size (100 ns units): 200 ms of headroom so a slow /// Shared-mode buffer size (100 ns units): 200 ms of headroom so a slow
/// disk write in the recorder does not overflow the capture buffer. /// disk write in the recorder does not overflow the capture buffer.
const BUFFER_DURATION_HNS: i64 = 2_000_000; const BUFFER_DURATION_HNS: i64 = 2_000_000;
@@ -57,7 +49,6 @@ impl Endpoint {
fn fallback(&self) -> Endpoint { fn fallback(&self) -> Endpoint {
match self { match self {
Endpoint::Mic(_) => Endpoint::Mic(None), Endpoint::Mic(_) => Endpoint::Mic(None),
Endpoint::Output(OutputTarget::Process { .. }) => self.clone(),
Endpoint::Output(_) => Endpoint::Output(OutputTarget::DefaultRender), Endpoint::Output(_) => Endpoint::Output(OutputTarget::DefaultRender),
} }
} }
@@ -188,40 +179,20 @@ fn device_by_id(enumerator: &DeviceEnumerator, id: &str) -> Result<Device, Captu
} }
/// Describes the endpoint `open` just opened, for the one-line log on a /// Describes the endpoint `open` just opened, for the one-line log on a
/// successful open. For process loopback that is the root pid and the exe /// successful open: the friendly name, the device id and whether it was
/// name of that process (looked up the same way call detection does); /// picked as the communications default or by an explicit id.
/// "unknown" if the process cannot be resolved. For a device endpoint it is fn describe_endpoint(device: &Device, role: &str) -> String {
/// the friendly name, the device id and whether it was picked as the
/// communications default or by an explicit id.
fn describe_endpoint(endpoint: &Endpoint, device: Option<&Device>, role: &str) -> String {
match endpoint {
Endpoint::Output(OutputTarget::Process { root_pid }) => {
let exe = sessions::image_name(*root_pid).unwrap_or_else(|| "unknown".to_string());
format!("process tree pid {root_pid} ({exe})")
}
_ => {
let device = device.expect("device endpoint always has a Device");
let name = device let name = device
.get_friendlyname() .get_friendlyname()
.unwrap_or_else(|_| "unknown".to_string()); .unwrap_or_else(|_| "unknown".to_string());
let id = device.get_id().unwrap_or_else(|_| "unknown".to_string()); let id = device.get_id().unwrap_or_else(|_| "unknown".to_string());
format!("{name} ({id}, {role})") format!("{name} ({id}, {role})")
} }
}
}
/// Opens `endpoint` and starts streaming. `format` forces a stream format /// Opens `endpoint` and starts streaming. `format` forces a stream format
/// (used when reopening); otherwise the endpoint's mix format is used, with /// (used when reopening); otherwise the endpoint's mix format is used, with
/// the samples converted to `f32` by the audio engine. /// the samples converted to `f32` by the audio engine.
fn open(endpoint: &Endpoint, format: Option<StreamFormat>) -> Result<Stream, CaptureError> { fn open(endpoint: &Endpoint, format: Option<StreamFormat>) -> Result<Stream, CaptureError> {
let (mut client, description) = match endpoint {
Endpoint::Output(OutputTarget::Process { root_pid }) => {
let client =
AudioClient::new_application_loopback_client(*root_pid, true).map_err(backend)?;
let description = describe_endpoint(endpoint, None, "");
(client, description)
}
_ => {
let enumerator = DeviceEnumerator::new().map_err(backend)?; let enumerator = DeviceEnumerator::new().map_err(backend)?;
let (device, role) = match endpoint { let (device, role) = match endpoint {
Endpoint::Mic(Some(id)) | Endpoint::Output(OutputTarget::Device(id)) => { Endpoint::Mic(Some(id)) | Endpoint::Output(OutputTarget::Device(id)) => {
@@ -233,22 +204,19 @@ fn open(endpoint: &Endpoint, format: Option<StreamFormat>) -> Result<Stream, Cap
.map_err(backend)?, .map_err(backend)?,
"communications default", "communications default",
), ),
Endpoint::Output(_) => ( Endpoint::Output(OutputTarget::DefaultRender) => (
enumerator enumerator
.get_default_device_for_role(&Direction::Render, &Role::Communications) .get_default_device_for_role(&Direction::Render, &Role::Communications)
.map_err(backend)?, .map_err(backend)?,
"communications default", "communications default",
), ),
}; };
let description = describe_endpoint(endpoint, Some(&device), role); let description = describe_endpoint(&device, role);
(device.get_iaudioclient().map_err(backend)?, description) let mut client = device.get_iaudioclient().map_err(backend)?;
}
};
let format = match (format, endpoint) { let format = match format {
(Some(format), _) => format, Some(format) => format,
(None, Endpoint::Output(OutputTarget::Process { .. })) => PROCESS_LOOPBACK_FORMAT, None => {
(None, _) => {
let mix = client.get_mixformat().map_err(backend)?; let mix = client.get_mixformat().map_err(backend)?;
StreamFormat { StreamFormat {
rate: mix.get_samplespersec(), rate: mix.get_samplespersec(),
@@ -268,8 +236,8 @@ fn open(endpoint: &Endpoint, format: Option<StreamFormat>) -> Result<Stream, Cap
autoconvert: true, autoconvert: true,
buffer_duration_hns: BUFFER_DURATION_HNS, buffer_duration_hns: BUFFER_DURATION_HNS,
}; };
// Capture on a capture endpoint records it; on a render endpoint (and on // Capture on a capture endpoint records it; on a render endpoint it sets
// the application loopback client) it sets up loopback capture. // up loopback capture.
client client
.initialize_client(&wave_format, &Direction::Capture, &mode) .initialize_client(&wave_format, &Direction::Capture, &mode)
.map_err(backend)?; .map_err(backend)?;
+91 -25
View File
@@ -1,18 +1,21 @@
//! Which processes hold a microphone right now (drives call detection). //! Which processes hold a microphone (drives call detection) and which play
//! on which output device (chooses the device a detected call is recorded
//! from).
//! //!
//! Every active capture endpoint is asked for its audio sessions //! Every active endpoint of a direction is asked for its audio sessions
//! (`IAudioSessionManager2::GetSessionEnumerator`). A session belongs to one //! (`IAudioSessionManager2::GetSessionEnumerator`). A session belongs to one
//! process (`IAudioSessionControl2::GetProcessId`) and is active while one //! process (`IAudioSessionControl2::GetProcessId`) and is active while one
//! of its streams runs, i.e. while the app actually records the mic. The //! of its streams runs, i.e. while the app actually records the mic or plays
//! process's exe comes from `QueryFullProcessImageNameW`; its `root_pid` is //! sound. The process's exe comes from `QueryFullProcessImageNameW`; its
//! the top of its same-exe parent chain in a ToolHelp snapshot, which is the //! `root_pid` is the top of its same-exe parent chain in a ToolHelp
//! tree application loopback records. Teams helper processes are reported //! snapshot. Teams helper processes are reported as their Teams ancestor
//! as their Teams ancestor (see [`crate::process_tree`]). //! (see [`crate::process_tree`]).
use std::path::Path; use std::path::Path;
use std::thread;
use recorder_core::detect::MicSession; use recorder_core::detect::MicSession;
use wasapi::{deinitialize, initialize_mta, DeviceEnumerator, Direction, SessionState}; use wasapi::{deinitialize, initialize_mta, Device, DeviceEnumerator, Direction, SessionState};
use windows::core::PWSTR; use windows::core::PWSTR;
use windows::Win32::Foundation::{CloseHandle, FILETIME, HANDLE}; use windows::Win32::Foundation::{CloseHandle, FILETIME, HANDLE};
use windows::Win32::System::Diagnostics::ToolHelp::{ use windows::Win32::System::Diagnostics::ToolHelp::{
@@ -24,6 +27,7 @@ use windows::Win32::System::Threading::{
}; };
use crate::process_tree::{attribute, ProcessTable}; use crate::process_tree::{attribute, ProcessTable};
use crate::render_sessions::RenderSession;
/// Sessions on every active capture endpoint, one entry per session. /// Sessions on every active capture endpoint, one entry per session.
/// ///
@@ -37,24 +41,45 @@ pub fn mic_sessions() -> Vec<MicSession> {
log::debug!("mic sessions: COM initialisation failed: {err}"); log::debug!("mic sessions: COM initialisation failed: {err}");
return Vec::new(); return Vec::new();
} }
let sessions = enumerate(); let sessions = enumerate_mic();
// Every COM object is dropped by now. // Every COM object is dropped by now.
deinitialize(); deinitialize();
sessions sessions
} }
/// (pid, active) of every non-system session on every active capture /// Sessions on every active render endpoint, in endpoint enumeration
/// endpoint, then resolved to exe and root pid. /// order, one entry per session.
fn enumerate() -> Vec<MicSession> { ///
let raw = capture_sessions(); /// Runs on a fresh thread so COM is initialised the way this code expects,
/// independent of the caller's apartment. Skips the same sessions as
/// [`mic_sessions`]; errors yield fewer or no sessions.
pub fn render_sessions() -> Vec<RenderSession> {
thread::spawn(|| {
if let Err(err) = initialize_mta().ok() {
log::debug!("render sessions: COM initialisation failed: {err}");
return Vec::new();
}
let sessions = enumerate_render();
// Every COM object is dropped by now.
deinitialize();
sessions
})
.join()
.unwrap_or_default()
}
fn enumerate_mic() -> Vec<MicSession> {
let raw: Vec<(u32, bool)> = endpoint_sessions(&Direction::Capture)
.into_iter()
.flat_map(|(_, sessions)| sessions)
.collect();
if raw.is_empty() { if raw.is_empty() {
return Vec::new(); return Vec::new();
} }
let table = process_table(); let table = process_table();
raw.into_iter() raw.into_iter()
.filter_map(|(pid, active)| { .filter_map(|(pid, active)| {
let exe = image_name(pid)?; let (exe, root_pid) = resolve(&table, pid)?;
let (exe, root_pid) = attribute(&table, pid, &exe, creation_time);
Some(MicSession { Some(MicSession {
exe, exe,
pid, pid,
@@ -65,17 +90,57 @@ fn enumerate() -> Vec<MicSession> {
.collect() .collect()
} }
fn capture_sessions() -> Vec<(u32, bool)> { fn enumerate_render() -> Vec<RenderSession> {
let endpoints = endpoint_sessions(&Direction::Render);
if endpoints.iter().all(|(_, sessions)| sessions.is_empty()) {
return Vec::new();
}
let table = process_table();
let mut result = Vec::new();
for (device, sessions) in endpoints {
let Ok(device_id) = device.get_id() else {
continue;
};
let device_name = device
.get_friendlyname()
.unwrap_or_else(|_| "unknown".to_string());
for (pid, active) in sessions {
let Some((exe, root_pid)) = resolve(&table, pid) else {
continue;
};
result.push(RenderSession {
exe,
pid,
root_pid,
active,
device_id: device_id.clone(),
device_name: device_name.clone(),
});
}
}
result
}
/// The app `pid` belongs to, `(exe, root_pid)`; `None` if the process
/// cannot be opened.
fn resolve(table: &ProcessTable, pid: u32) -> Option<(String, u32)> {
let exe = image_name(pid)?;
Some(attribute(table, pid, &exe, creation_time))
}
/// Every active endpoint of `direction` with the (pid, active) of each of
/// its non-system sessions.
fn endpoint_sessions(direction: &Direction) -> Vec<(Device, Vec<(u32, bool)>)> {
let collection = match DeviceEnumerator::new() let collection = match DeviceEnumerator::new()
.and_then(|enumerator| enumerator.get_device_collection(&Direction::Capture)) .and_then(|enumerator| enumerator.get_device_collection(direction))
{ {
Ok(collection) => collection, Ok(collection) => collection,
Err(err) => { Err(err) => {
log::debug!("mic sessions: cannot enumerate capture endpoints: {err}"); log::debug!("audio sessions: cannot enumerate {direction} endpoints: {err}");
return Vec::new(); return Vec::new();
} }
}; };
let mut sessions = Vec::new(); let mut endpoints = Vec::new();
for device in &collection { for device in &collection {
let Ok(device) = device else { continue }; let Ok(device) = device else { continue };
let Ok(enumerator) = device let Ok(enumerator) = device
@@ -84,6 +149,7 @@ fn capture_sessions() -> Vec<(u32, bool)> {
else { else {
continue; continue;
}; };
let mut sessions = Vec::new();
let count = enumerator.get_count().unwrap_or(0); let count = enumerator.get_count().unwrap_or(0);
for index in 0..count { for index in 0..count {
let Ok(control) = enumerator.get_session(index) else { let Ok(control) = enumerator.get_session(index) else {
@@ -96,14 +162,14 @@ fn capture_sessions() -> Vec<(u32, bool)> {
sessions.push((pid, state == SessionState::Active)); sessions.push((pid, state == SessionState::Active));
} }
} }
endpoints.push((device, sessions));
} }
sessions endpoints
} }
/// The file name of `pid`'s executable, or `None` if the process cannot be /// The file name of `pid`'s executable, or `None` if the process cannot be
/// opened (exited, or protected). Also used by `windows::capture` to name /// opened (exited, or protected).
/// the process behind a process-loopback endpoint in its diagnostic log. fn image_name(pid: u32) -> Option<String> {
pub(crate) fn image_name(pid: u32) -> Option<String> {
// SAFETY: plain Win32 calls; the handle is closed below and the buffer // SAFETY: plain Win32 calls; the handle is closed below and the buffer
// outlives the call that fills it. // outlives the call that fills it.
unsafe { unsafe {
@@ -155,7 +221,7 @@ fn process_table() -> ProcessTable {
let snapshot = match CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) { let snapshot = match CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) {
Ok(snapshot) => snapshot, Ok(snapshot) => snapshot,
Err(err) => { Err(err) => {
log::debug!("mic sessions: process snapshot failed: {err}"); log::debug!("audio sessions: process snapshot failed: {err}");
return table; return table;
} }
}; };
@@ -188,6 +254,6 @@ fn close(handle: HANDLE) {
// SAFETY: `handle` was returned by a successful Win32 call and is closed // SAFETY: `handle` was returned by a successful Win32 call and is closed
// exactly once. // exactly once.
if let Err(err) = unsafe { CloseHandle(handle) } { if let Err(err) = unsafe { CloseHandle(handle) } {
log::debug!("mic sessions: CloseHandle failed: {err}"); log::debug!("audio sessions: CloseHandle failed: {err}");
} }
} }
-4
View File
@@ -18,10 +18,6 @@ pub struct StreamFormat {
/// What the output ("out") track records. /// What the output ("out") track records.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum OutputTarget { pub enum OutputTarget {
/// Only the audio rendered by this process and its children (Windows
/// application loopback). Falls back to [`OutputTarget::DefaultRender`]
/// where per-process capture is not available.
Process { root_pid: u32 },
/// Everything played on the default (communications) output device. /// Everything played on the default (communications) output device.
DefaultRender, DefaultRender,
/// Everything played on the output device with this id. /// Everything played on the output device with this id.
+5 -4
View File
@@ -213,12 +213,13 @@ pub fn start_manual(app: &AppHandle) -> CmdResult<CallId> {
Ok(call_id) Ok(call_id)
} }
/// Starts recording a call that detection found (`app` is its display /// Starts recording a call that detection found (`name` is its display
/// name, `root_pid` the process tree whose audio is the far end). /// name, `exe` and `root_pid` identify the call app, whose output device
pub fn start_detected(app: &AppHandle, name: &str, root_pid: u32) -> CmdResult<CallId> { /// is recorded).
pub fn start_detected(app: &AppHandle, name: &str, exe: &str, root_pid: u32) -> CmdResult<CallId> {
let call_id = app let call_id = app
.state::<AppState>() .state::<AppState>()
.start_detected_recording(name, root_pid)?; .start_detected_recording(name, exe, root_pid)?;
on_started(app, call_id); on_started(app, call_id);
Ok(call_id) Ok(call_id)
} }
+8 -3
View File
@@ -54,6 +54,7 @@ enum Action {
/// Record the detected call. /// Record the detected call.
Start { Start {
app: String, app: String,
exe: String,
root_pid: u32, root_pid: u32,
}, },
/// A manual recording already covers the call: only keep the detector /// A manual recording already covers the call: only keep the detector
@@ -67,7 +68,9 @@ enum Action {
fn decide(event: Option<DetectorEvent>, manual_recording: bool) -> Action { fn decide(event: Option<DetectorEvent>, manual_recording: bool) -> Action {
match event { match event {
Some(DetectorEvent::CallStarted { .. }) if manual_recording => Action::Suppress, Some(DetectorEvent::CallStarted { .. }) if manual_recording => Action::Suppress,
Some(DetectorEvent::CallStarted { app, root_pid, .. }) => Action::Start { app, root_pid }, Some(DetectorEvent::CallStarted { exe, app, root_pid }) => {
Action::Start { app, exe, root_pid }
}
Some(DetectorEvent::CallEnded) => Action::EndDetected, Some(DetectorEvent::CallEnded) => Action::EndDetected,
None => Action::Nothing, None => Action::Nothing,
} }
@@ -85,14 +88,15 @@ fn poll(app: &AppHandle, last_sessions: &mut String) {
} }
let event = state.detector_tick(Instant::now(), &sessions); let event = state.detector_tick(Instant::now(), &sessions);
if let Some(DetectorEvent::CallStarted { exe, root_pid, .. }) = &event { if let Some(DetectorEvent::CallStarted { exe, root_pid, .. }) = &event {
log::info!("call detected: {exe} (process tree {root_pid})"); log::info!("call detected: {exe} (root pid {root_pid})");
} }
match decide(event, state.manual_recording_running()) { match decide(event, state.manual_recording_running()) {
Action::Start { Action::Start {
app: name, app: name,
exe,
root_pid, root_pid,
} => { } => {
if let Err(e) = commands::start_detected(app, &name, root_pid) { if let Err(e) = commands::start_detected(app, &name, &exe, root_pid) {
log::error!("recording the detected call failed: {e}"); log::error!("recording the detected call failed: {e}");
// Otherwise the detector would stay in the call and never // Otherwise the detector would stay in the call and never
// retry; suppressing waits for the app to release the mic. // retry; suppressing waits for the app to release the mic.
@@ -146,6 +150,7 @@ mod tests {
decide(Some(started()), false), decide(Some(started()), false),
Action::Start { Action::Start {
app: "Microsoft Teams".into(), app: "Microsoft Teams".into(),
exe: "ms-teams.exe".into(),
root_pid: 100 root_pid: 100
} }
); );
+2
View File
@@ -43,6 +43,8 @@ pub fn run() {
tauri_plugin_log::Builder::new() tauri_plugin_log::Builder::new()
.targets(targets) .targets(targets)
.level(log::LevelFilter::Info) .level(log::LevelFilter::Info)
// flacenc logs per-encode statistics at Info.
.level_for("flacenc", log::LevelFilter::Warn)
.max_file_size(LOG_FILE_SIZE) .max_file_size(LOG_FILE_SIZE)
.rotation_strategy(RotationStrategy::KeepSome(2)) .rotation_strategy(RotationStrategy::KeepSome(2))
.timezone_strategy(TimezoneStrategy::UseLocal) .timezone_strategy(TimezoneStrategy::UseLocal)
+94 -6
View File
@@ -16,6 +16,7 @@ use std::thread;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use chrono::{DateTime, Local}; use chrono::{DateTime, Local};
use recorder_audio::RenderSession;
use recorder_core::accounts; use recorder_core::accounts;
use recorder_core::call_view::{self, CallDetail, CallListItem}; use recorder_core::call_view::{self, CallDetail, CallListItem};
use recorder_core::capture::{CaptureSource, OutputTarget}; use recorder_core::capture::{CaptureSource, OutputTarget};
@@ -255,12 +256,27 @@ impl AppState {
self.start_recording(MANUAL_APP, false, out) self.start_recording(MANUAL_APP, false, out)
} }
/// Starts recording a detected call: the configured mic and the audio /// Starts recording a detected call (`app` is its display name, `exe`
/// of the call app's process tree (or, if that cannot be captured, the /// and `root_pid` identify the call app): the configured mic and the
/// default output). /// output device the call plays on (see [`detected_output_target`]).
pub fn start_detected_recording(&self, app: &str, root_pid: u32) -> Result<CallId, String> { pub fn start_detected_recording(
let out = recorder_audio::output_source_with_fallback(OutputTarget::Process { root_pid }); &self,
self.start_recording(app, true, out) app: &str,
exe: &str,
root_pid: u32,
) -> Result<CallId, String> {
let sessions = recorder_audio::render_sessions();
log::info!(
"render sessions: [{}]",
recorder_audio::describe_render_sessions(&sessions)
);
let target = detected_output_target(
self.settings().output_device.as_deref(),
&sessions,
exe,
root_pid,
);
self.start_recording(app, true, recorder_audio::output_source(target))
} }
/// Creates a call row and records the configured mic and `out` into /// Creates a call row and records the configured mic and `out` into
@@ -621,6 +637,37 @@ impl AppState {
} }
} }
/// The output a detected call is recorded from: the configured output
/// device, else the device on which the call app (`exe`, `root_pid`) has an
/// active render session, else the default communications device. Always
/// device loopback: new Teams plays call audio from a process outside the
/// ms-teams.exe tree, so loopback of that process tree recorded silence.
fn detected_output_target(
output_device: Option<&str>,
sessions: &[RenderSession],
exe: &str,
root_pid: u32,
) -> OutputTarget {
if let Some(id) = output_device {
log::info!("out track: configured output device {id}");
return OutputTarget::Device(id.to_string());
}
match recorder_audio::render_endpoint_for(sessions, exe, root_pid) {
Some(session) => {
log::info!(
"out track: call app playing on {} ({})",
session.device_name,
session.device_id
);
OutputTarget::Device(session.device_id.clone())
}
None => {
log::info!("out track: no render session of {exe} found, using communications default");
OutputTarget::DefaultRender
}
}
}
fn output_target(settings: &Settings) -> OutputTarget { fn output_target(settings: &Settings) -> OutputTarget {
match &settings.output_device { match &settings.output_device {
Some(id) => OutputTarget::Device(id.clone()), Some(id) => OutputTarget::Device(id.clone()),
@@ -1070,4 +1117,45 @@ mod tests {
assert_eq!(f.state.save_settings(valid.clone()), Ok(())); assert_eq!(f.state.save_settings(valid.clone()), Ok(()));
assert_eq!(f.state.settings(), valid); assert_eq!(f.state.settings(), valid);
} }
fn render_session(exe: &str, root_pid: u32, device: &str) -> RenderSession {
RenderSession {
exe: exe.to_string(),
pid: root_pid + 1,
root_pid,
active: true,
device_id: format!("id-{device}"),
device_name: device.to_string(),
}
}
#[test]
fn detected_calls_record_the_device_the_call_app_plays_on() {
let sessions = [
render_session("spotify.exe", 50, "Lautsprecher"),
render_session("ms-teams.exe", 100, "Headset"),
];
assert_eq!(
detected_output_target(None, &sessions, "ms-teams.exe", 100),
OutputTarget::Device("id-Headset".into())
);
}
#[test]
fn detected_calls_without_a_render_session_record_the_default_device() {
let sessions = [render_session("spotify.exe", 50, "Lautsprecher")];
assert_eq!(
detected_output_target(None, &sessions, "ms-teams.exe", 100),
OutputTarget::DefaultRender
);
}
#[test]
fn a_configured_output_device_wins_for_detected_calls() {
let sessions = [render_session("ms-teams.exe", 100, "Headset")];
assert_eq!(
detected_output_target(Some("id-Lautsprecher"), &sessions, "ms-teams.exe", 100),
OutputTarget::Device("id-Lautsprecher".into())
);
}
} }