diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index e8cde10223a..8fe766379d9 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -101,6 +101,9 @@ jobs: sudo apt install -qy --no-install-recommends mesa-vulkan-drivers fonts-noto-cjk fonts-dejavu-extra # FIXME #35029 sudo apt purge -y fonts-droid-fallback + sudo apt install -y jackd2 libjack-jackd2-0 + - name: Start Dummy Audio Device + run: jackd -d dummy & - name: Sync from upstream WPT if: ${{ inputs.wpt-sync-from-upstream }} run: | diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 814e20aa9c3..8f60bbd403e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -191,7 +191,7 @@ jobs: flags: unittests,unittests-linux,unittests-linux-${{ inputs.profile }} token: ${{ secrets.CODECOV_TOKEN }} - name: Devtools tests - if: ${{ false && inputs.unit-tests }} # FIXME #39273 + if: ${{ false && inputs.unit-tests }} # FIXME #39273 run: ./mach test-devtools --profile ${{ inputs.profile }} - name: Archive build timing uses: actions/upload-artifact@v6 @@ -231,7 +231,7 @@ jobs: if: ${{ inputs.bencher && inputs.profile != 'debug' && github.event_name != 'workflow_dispatch' && github.event_name != 'merge_group' }} uses: ./.github/workflows/bencher.yml with: - target: 'linux' + target: "linux" profile: ${{ inputs.profile }} compressed-file-path: ${{ inputs.profile }}-binary-linux/servo-tech-demo.tar.gz binary-path: servo/servo @@ -368,3 +368,65 @@ jobs: files: codecov.json,support/crown/codecov.json fail_ci_if_error: true flags: unittests + + media-examples: + name: Media Examples + if: ${{ inputs.unit-tests }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v5 + if: github.event_name != 'pull_request_target' + with: + fetch-depth: 1 + - name: Install Dependencies + run: | + sudo bash -c 'apt-add-repository -y https://mirrors.kernel.org/ubuntu' + sudo apt update + sudo apt install -y \ + gstreamer1.0-libav \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-tools \ + libasound2-plugins \ + libfaad2 \ + libffi7 \ + libfftw3-single3 \ + libges-1.0-dev \ + libgstreamer-plugins-bad1.0-dev\ + libgstreamer-plugins-base1.0-dev \ + libgstreamer1.0-dev \ + libgstrtspserver-1.0-dev \ + libjack-jackd2-0 \ + libmms0 \ + libmpg123-0 \ + libopus0 \ + liborc-0.4-0 \ + liborc-0.4-dev \ + libpulsedsp \ + libsamplerate0 \ + libspeexdsp1 \ + libtdb1 \ + libtheora0 \ + libtwolame0 \ + libwayland-egl1-mesa \ + libwebrtc-audio-processing1 \ + pulseaudio \ + pulseaudio-utils \ + webp + - name: Determine MSRV + id: msrv + uses: ./.github/actions/parse_msrv + - name: Install MSRV + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.msrv.outputs.rust_version }} + - name: Virtual Audio Devices + run: | + jackd -d dummy & + pulseaudio --start + gst-inspect-1.0 | grep Total + - name: Run Examples + run: | + ls components/media/examples/examples/*.rs | xargs -I{} basename {} .rs | grep -v params_connect | RUST_BACKTRACE=1 GST_DEBUG=3 xargs -I{} cargo run -p media-examples --example {} diff --git a/Cargo.lock b/Cargo.lock index dffd326d510..829e12c3c08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3583,6 +3583,56 @@ dependencies = [ "system-deps 7.0.5", ] +[[package]] +name = "gstreamer-gl-wayland" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7c882c6de202d985302f531ee7f907e1f1a89b11caa1ba8a153764dc6c650c" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-gl", + "gstreamer-gl-wayland-sys", + "libc", +] + +[[package]] +name = "gstreamer-gl-wayland-sys" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8dffafb60f97705cb2395139df2002fa2730fa75d4efa04ed1ece1a014d464" +dependencies = [ + "glib-sys", + "gstreamer-gl-sys", + "libc", + "system-deps 7.0.5", +] + +[[package]] +name = "gstreamer-gl-x11" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a5cbe1aaf44cf6af4beef95390f3be8fdf919059a26991c922dcb0200c21535" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-gl", + "gstreamer-gl-x11-sys", + "libc", +] + +[[package]] +name = "gstreamer-gl-x11-sys" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be689b7d327607d11336964e4dc0f5da658b3a4d0373f0e267995edbe0e0454c" +dependencies = [ + "glib-sys", + "gstreamer-gl-sys", + "libc", + "system-deps 7.0.5", +] + [[package]] name = "gstreamer-play" version = "0.24.4" @@ -5349,6 +5399,18 @@ dependencies = [ "webrender_api", ] +[[package]] +name = "media-examples" +version = "0.0.1" +dependencies = [ + "euclid", + "rand 0.9.2", + "serde", + "servo-media", + "servo-media-auto", + "servo-media-dummy", +] + [[package]] name = "memchr" version = "2.7.6" @@ -8238,8 +8300,7 @@ dependencies = [ [[package]] name = "servo-media" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "once_cell", "servo-media-audio", @@ -8251,8 +8312,7 @@ dependencies = [ [[package]] name = "servo-media-audio" -version = "0.2.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "byte-slice-cast", "euclid", @@ -8270,10 +8330,17 @@ dependencies = [ "speexdsp-resampler", ] +[[package]] +name = "servo-media-auto" +version = "0.0.1" +dependencies = [ + "servo-media-dummy", + "servo-media-gstreamer", +] + [[package]] name = "servo-media-derive" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "proc-macro2", "quote", @@ -8282,8 +8349,7 @@ dependencies = [ [[package]] name = "servo-media-dummy" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "ipc-channel", "servo-media", @@ -8296,8 +8362,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "byte-slice-cast", "glib", @@ -8329,8 +8394,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "gstreamer", "gstreamer-video", @@ -8339,8 +8403,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render-android" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "glib", "gstreamer", @@ -8353,13 +8416,14 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render-unix" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "glib", "gstreamer", "gstreamer-gl", "gstreamer-gl-egl", + "gstreamer-gl-wayland", + "gstreamer-gl-x11", "gstreamer-video", "servo-media-gstreamer-render", "servo-media-player", @@ -8367,8 +8431,7 @@ dependencies = [ [[package]] name = "servo-media-player" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "ipc-channel", "serde", @@ -8379,21 +8442,18 @@ dependencies = [ [[package]] name = "servo-media-streams" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "uuid", ] [[package]] name = "servo-media-traits" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" [[package]] name = "servo-media-webrtc" -version = "0.1.0" -source = "git+https://github.com/servo/media?rev=f384dbc4ff8b5c6f8db2c763306cbe2281d66391#f384dbc4ff8b5c6f8db2c763306cbe2281d66391" +version = "0.0.1" dependencies = [ "log", "servo-media-streams", diff --git a/Cargo.toml b/Cargo.toml index 226458ef92b..708cdffeacf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "components/media/examples", "components/xpath", "ports/servoshell", "tests/unit/*", @@ -74,13 +75,23 @@ flate2 = "1.1" fonts_traits = { path = "components/shared/fonts" } freetype-sys = "0.20" gleam = "0.15" +glib = "0.21" +glib-sys = "0.21" glow = "0.16.0" gstreamer = { version = "0.24", features = ["v1_18"] } +gstreamer-app = "0.24" +gstreamer-audio = "0.24" gstreamer-base = "0.24" gstreamer-gl = "0.24" +gstreamer-gl-egl = "0.24" gstreamer-gl-sys = "0.24" +gstreamer-gl-wayland = "0.24" +gstreamer-gl-x11 = "0.24" +gstreamer-play = "0.24" +gstreamer-sdp = "0.24" gstreamer-sys = "0.24" gstreamer-video = "0.24" +gstreamer-webrtc = { version = "0.24", features = ["v1_18"] } harfbuzz-sys = "0.6.1" headers = "0.4" hitrace = "0.1.6" @@ -162,9 +173,9 @@ serde = "1.0.228" serde_bytes = "0.11" serde_core = "1.0.226" serde_json = "1.0" -servo-media = { git = "https://github.com/servo/media", rev = "f384dbc4ff8b5c6f8db2c763306cbe2281d66391" } -servo-media-dummy = { git = "https://github.com/servo/media", rev = "f384dbc4ff8b5c6f8db2c763306cbe2281d66391" } -servo-media-gstreamer = { git = "https://github.com/servo/media", rev = "f384dbc4ff8b5c6f8db2c763306cbe2281d66391" } +servo-media = { path = "components/media/servo-media" } +servo-media-dummy = { path = "components/media/backends/dummy" } +servo-media-gstreamer = { path = "components/media/backends/gstreamer" } servo-tracing = { path = "components/servo_tracing" } servo_arc = { git = "https://github.com/servo/stylo", rev = "3b46f3ef27eb14ca276b4bf3ff041a6c43d93827" } sha1 = "0.10" diff --git a/components/constellation/Cargo.toml b/components/constellation/Cargo.toml index 723c1fc6c56..c520b7251a5 100644 --- a/components/constellation/Cargo.toml +++ b/components/constellation/Cargo.toml @@ -42,7 +42,7 @@ ipc-channel = { workspace = true } keyboard-types = { workspace = true } layout_api = { workspace = true } log = { workspace = true } -media = { path = "../media" } +media = { path = "../media/media-thread" } net = { path = "../net" } net_traits = { workspace = true } paint_api = { workspace = true } diff --git a/components/media/audio/Cargo.toml b/components/media/audio/Cargo.toml new file mode 100644 index 00000000000..d046868b216 --- /dev/null +++ b/components/media/audio/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "servo-media-audio" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_audio" +path = "lib.rs" + +[dependencies] +euclid = "0.22" +log = "0.4" +serde_derive = "1.0.66" +serde = "1.0.66" +servo-media-derive = { path = "../servo-media-derive" } +servo-media-player = { path = "../player" } +servo-media-traits = { path = "../traits" } +servo-media-streams = { path = "../streams" } +smallvec = "1" +speexdsp-resampler = "0.1.0" +num-complex = "0.2.4" + +[dependencies.petgraph] +version = "0.4.12" +features = ["stable_graph"] + +[dependencies.byte-slice-cast] +version = "1" + +[dependencies.num-traits] +version = "0.2" diff --git a/components/media/audio/analyser_node.rs b/components/media/audio/analyser_node.rs new file mode 100644 index 00000000000..cc26ecbfbb6 --- /dev/null +++ b/components/media/audio/analyser_node.rs @@ -0,0 +1,283 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cmp; +use std::f32::consts::PI; + +use crate::block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE}; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo, ChannelInterpretation}; + +#[derive(AudioNodeCommon)] +pub(crate) struct AnalyserNode { + channel_info: ChannelInfo, + callback: Box, +} + +impl AnalyserNode { + pub fn new(callback: Box, channel_info: ChannelInfo) -> Self { + Self { + callback, + channel_info, + } + } +} + +impl AudioNodeEngine for AnalyserNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::AnalyserNode + } + + fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + let mut push = inputs.blocks[0].clone(); + push.mix(1, ChannelInterpretation::Speakers); + + (self.callback)(push); + + // analyser node doesn't modify the inputs + inputs + } +} + +/// From +pub const MAX_FFT_SIZE: usize = 32768; +pub const MAX_BLOCK_COUNT: usize = MAX_FFT_SIZE / FRAMES_PER_BLOCK_USIZE; + +/// The actual analysis is done on the DOM side. We provide +/// the actual base functionality in this struct, so the DOM +/// just has to do basic shimming +pub struct AnalysisEngine { + /// The number of past sample-frames to consider in the FFT + fft_size: usize, + smoothing_constant: f64, + min_decibels: f64, + max_decibels: f64, + /// This is a ring buffer containing the last MAX_FFT_SIZE + /// sample-frames + data: Box<[f32; MAX_FFT_SIZE]>, + /// The index of the current block + current_block: usize, + /// Have we computed the FFT already? + fft_computed: bool, + /// Cached blackman window data + blackman_windows: Vec, + /// The smoothed FFT data (in frequency domain) + smoothed_fft_data: Vec, + /// The computed FFT data, in decibels + computed_fft_data: Vec, + /// The windowed time domain data + /// Used during FFT computation + windowed: Vec, +} + +impl AnalysisEngine { + pub fn new( + fft_size: usize, + smoothing_constant: f64, + min_decibels: f64, + max_decibels: f64, + ) -> Self { + debug_assert!((32..=32768).contains(&fft_size)); + // must be a power of two + debug_assert!(fft_size & (fft_size - 1) == 0); + debug_assert!((0. ..=1.).contains(&smoothing_constant)); + debug_assert!(max_decibels > min_decibels); + Self { + fft_size, + smoothing_constant, + min_decibels, + max_decibels, + data: Box::new([0.; MAX_FFT_SIZE]), + current_block: MAX_BLOCK_COUNT - 1, + fft_computed: false, + blackman_windows: Vec::with_capacity(fft_size), + computed_fft_data: Vec::with_capacity(fft_size / 2), + smoothed_fft_data: Vec::with_capacity(fft_size / 2), + windowed: Vec::with_capacity(fft_size), + } + } + + pub fn set_fft_size(&mut self, fft_size: usize) { + debug_assert!((32..=32768).contains(&fft_size)); + // must be a power of two + debug_assert!(fft_size & (fft_size - 1) == 0); + self.fft_size = fft_size; + self.fft_computed = false; + } + + pub fn get_fft_size(&self) -> usize { + self.fft_size + } + + pub fn set_smoothing_constant(&mut self, smoothing_constant: f64) { + debug_assert!((0. ..=1.).contains(&smoothing_constant)); + self.smoothing_constant = smoothing_constant; + self.fft_computed = false; + } + + pub fn get_smoothing_constant(&self) -> f64 { + self.smoothing_constant + } + + pub fn set_min_decibels(&mut self, min_decibels: f64) { + debug_assert!(min_decibels < self.max_decibels); + self.min_decibels = min_decibels; + } + + pub fn get_min_decibels(&self) -> f64 { + self.min_decibels + } + + pub fn set_max_decibels(&mut self, max_decibels: f64) { + debug_assert!(self.min_decibels < max_decibels); + self.max_decibels = max_decibels; + } + + pub fn get_max_decibels(&self) -> f64 { + self.max_decibels + } + + fn advance(&mut self) { + self.current_block += 1; + if self.current_block >= MAX_BLOCK_COUNT { + self.current_block = 0; + } + } + + /// Get the data of the current block + fn curent_block_mut(&mut self) -> &mut [f32] { + let index = FRAMES_PER_BLOCK_USIZE * self.current_block; + &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)] + } + + /// Given an index from 0 to fft_size, convert it into an index into + /// the backing array + fn convert_index(&self, index: usize) -> usize { + let offset = self.fft_size - index; + let last_element = (1 + self.current_block) * FRAMES_PER_BLOCK_USIZE - 1; + if offset > last_element { + MAX_FFT_SIZE - offset + last_element + } else { + last_element - offset + } + } + + /// Given an index into the backing array, increment it + fn advance_index(&self, index: &mut usize) { + *index += 1; + if *index >= MAX_FFT_SIZE { + *index = 0; + } + } + + pub fn push(&mut self, mut block: Block) { + debug_assert!(block.chan_count() == 1); + self.advance(); + if !block.is_silence() { + self.curent_block_mut().copy_from_slice(block.data_mut()); + } + self.fft_computed = false; + } + + /// + fn compute_blackman_windows(&mut self) { + if self.blackman_windows.len() == self.fft_size { + return; + } + const ALPHA: f32 = 0.16; + const ALPHA_0: f32 = (1. - ALPHA) / 2.; + const ALPHA_1: f32 = 1. / 2.; + const ALPHA_2: f32 = ALPHA / 2.; + self.blackman_windows.resize(self.fft_size, 0.); + let coeff = PI * 2. / self.fft_size as f32; + for n in 0..self.fft_size { + self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() + + ALPHA_2 * (2. * coeff * n as f32).cos(); + } + } + + fn apply_blackman_window(&mut self) { + self.compute_blackman_windows(); + self.windowed.resize(self.fft_size, 0.); + + let mut data_idx = self.convert_index(0); + for n in 0..self.fft_size { + self.windowed[n] = self.blackman_windows[n] * self.data[data_idx]; + self.advance_index(&mut data_idx); + } + } + + fn compute_fft(&mut self) { + if self.fft_computed { + return; + } + self.fft_computed = true; + self.apply_blackman_window(); + self.computed_fft_data.resize(self.fft_size / 2, 0.); + self.smoothed_fft_data.resize(self.fft_size / 2, 0.); + + for k in 0..(self.fft_size / 2) { + let mut sum_real = 0.; + let mut sum_imaginary = 0.; + let factor = -2. * PI * k as f32 / self.fft_size as f32; + for n in 0..(self.fft_size) { + sum_real += self.windowed[n] * (factor * n as f32).cos(); + sum_imaginary += self.windowed[n] * (factor * n as f32).sin(); + } + let sum_real = sum_real / self.fft_size as f32; + let sum_imaginary = sum_imaginary / self.fft_size as f32; + let magnitude = (sum_real * sum_real + sum_imaginary * sum_imaginary).sqrt(); + self.smoothed_fft_data[k] = (self.smoothing_constant * self.smoothed_fft_data[k] as f64 + + (1. - self.smoothing_constant) * magnitude as f64) + as f32; + self.computed_fft_data[k] = 20. * self.smoothed_fft_data[k].log(10.); + } + } + + pub fn fill_time_domain_data(&self, dest: &mut [f32]) { + let mut data_idx = self.convert_index(0); + let end = cmp::min(self.fft_size, dest.len()); + for entry in &mut dest[0..end] { + *entry = self.data[data_idx]; + self.advance_index(&mut data_idx); + } + } + + pub fn fill_byte_time_domain_data(&self, dest: &mut [u8]) { + let mut data_idx = self.convert_index(0); + let end = cmp::min(self.fft_size, dest.len()); + for entry in &mut dest[0..end] { + let result = 128. * (1. + self.data[data_idx]); + *entry = clamp_255(result); + self.advance_index(&mut data_idx) + } + } + + pub fn fill_frequency_data(&mut self, dest: &mut [f32]) { + self.compute_fft(); + let len = cmp::min(dest.len(), self.computed_fft_data.len()); + dest[0..len].copy_from_slice(&self.computed_fft_data[0..len]); + } + + pub fn fill_byte_frequency_data(&mut self, dest: &mut [u8]) { + self.compute_fft(); + let len = cmp::min(dest.len(), self.computed_fft_data.len()); + let ratio = 255. / (self.max_decibels - self.min_decibels); + for (index, freq) in dest[0..len].iter_mut().enumerate() { + let result = ratio * (self.computed_fft_data[index] as f64 - self.min_decibels); + *freq = clamp_255(result as f32); + } + } +} + +fn clamp_255(val: f32) -> u8 { + if val > 255. { + 255 + } else if val < 0. { + 0 + } else { + val as u8 + } +} diff --git a/components/media/audio/biquad_filter_node.rs b/components/media/audio/biquad_filter_node.rs new file mode 100644 index 00000000000..4e15ed22f11 --- /dev/null +++ b/components/media/audio/biquad_filter_node.rs @@ -0,0 +1,414 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::f64::consts::{PI, SQRT_2}; + +use smallvec::SmallVec; + +use crate::block::{Chunk, Tick}; +use crate::node::{AudioNodeEngine, AudioNodeMessage, AudioNodeType, BlockInfo, ChannelInfo}; +use crate::param::{Param, ParamType}; + +#[derive(Copy, Clone, Debug)] +pub struct BiquadFilterNodeOptions { + pub filter: FilterType, + pub frequency: f32, + pub detune: f32, + pub q: f32, + pub gain: f32, +} + +#[derive(Copy, Clone, Debug)] +pub enum FilterType { + LowPass, + HighPass, + BandPass, + LowShelf, + HighShelf, + Peaking, + Notch, + AllPass, +} + +impl Default for BiquadFilterNodeOptions { + fn default() -> Self { + BiquadFilterNodeOptions { + filter: FilterType::LowPass, + frequency: 350., + detune: 0., + q: 1., + gain: 0., + } + } +} + +#[derive(Copy, Clone, Debug)] +pub enum BiquadFilterNodeMessage { + SetFilterType(FilterType), +} + +/// The last two input and output values, per-channel +// Default sets all fields to zero +#[derive(Default, Copy, Clone, PartialEq)] +struct BiquadState { + /// The input value from last frame + x1: f64, + /// The input value from two frames ago + x2: f64, + /// The output value from last frame + y1: f64, + /// The output value from two frames ago + y2: f64, +} + +impl BiquadState { + /// Update with new input/output values from this frame + fn update(&mut self, x: f64, y: f64) { + self.x2 = self.x1; + self.x1 = x; + self.y2 = self.y1; + self.y1 = y; + } +} + +/// +#[derive(AudioNodeCommon)] +pub(crate) struct BiquadFilterNode { + channel_info: ChannelInfo, + filter: FilterType, + frequency: Param, + detune: Param, + q: Param, + gain: Param, + /// The computed filter parameter b0 + /// This is actually b0 / a0, we pre-divide + /// for efficiency + b0: f64, + /// The computed filter parameter b1 + /// This is actually b1 / a0, we pre-divide + /// for efficiency + b1: f64, + /// The computed filter parameter b2 + /// This is actually b2 / a0, we pre-divide + /// for efficiency + b2: f64, + /// The computed filter parameter a1 + /// This is actually a1 / a0, we pre-divide + /// for efficiency + a1: f64, + /// The computed filter parameter a2 + /// This is actually a2 / a0, we pre-divide + /// for efficiency + a2: f64, + /// Stored filter state, this contains the last two + /// frames of input and output values for every + /// channel + state: SmallVec<[BiquadState; 2]>, +} + +impl BiquadFilterNode { + pub fn new( + options: BiquadFilterNodeOptions, + channel_info: ChannelInfo, + sample_rate: f32, + ) -> Self { + let mut ret = Self { + channel_info, + filter: options.filter, + frequency: Param::new(options.frequency), + gain: Param::new(options.gain), + q: Param::new(options.q), + detune: Param::new(options.detune), + b0: 0., + b1: 0., + b2: 0., + a1: 0., + a2: 0., + state: SmallVec::new(), + }; + ret.update_coefficients(sample_rate); + ret + } + + pub fn update_parameters(&mut self, info: &BlockInfo, tick: Tick) -> bool { + let mut changed = self.frequency.update(info, tick); + changed |= self.detune.update(info, tick); + changed |= self.q.update(info, tick); + changed |= self.gain.update(info, tick); + + if changed { + self.update_coefficients(info.sample_rate); + } + changed + } + + /// Set to the constant z-transform y[n] = b0 * x[n] + fn constant_z_transform(&mut self, b0: f64) { + self.b0 = b0; + self.b1 = 0.; + self.b2 = 0.; + self.a1 = 0.; + self.a2 = 0.; + } + + /// Update the coefficients a1, a2, b0, b1, b2, given the sample_rate + /// + /// See + fn update_coefficients(&mut self, fs: f32) { + let g: f64 = self.gain.value().into(); + let q: f64 = self.q.value().into(); + let freq: f64 = self.frequency.value().into(); + let f0: f64 = freq * (2.0_f64).powf(self.detune.value() as f64 / 1200.); + let fs: f64 = fs.into(); + // clamp to nominal range + // https://webaudio.github.io/web-audio-api/#biquadfilternode + let f0 = if f0 > fs / 2. || !f0.is_finite() { + fs / 2. + } else if f0 < 0. { + 0. + } else { + f0 + }; + + let normalized = f0 / fs; + let a = 10.0_f64.powf(g / 40.); + + // the boundary values sometimes need limits to + // be taken + match self.filter { + FilterType::LowPass => { + if normalized == 1. { + self.constant_z_transform(1.); + return; + } else if normalized == 0. { + self.constant_z_transform(0.); + return; + } + }, + FilterType::HighPass => { + if normalized == 1. { + self.constant_z_transform(0.); + return; + } else if normalized == 0. { + self.constant_z_transform(1.); + return; + } + }, + FilterType::LowShelf => { + if normalized == 1. { + self.constant_z_transform(a * a); + return; + } else if normalized == 0. { + self.constant_z_transform(1.); + return; + } + }, + FilterType::HighShelf => { + if normalized == 1. { + self.constant_z_transform(1.); + return; + } else if normalized == 0. { + self.constant_z_transform(a * a); + return; + } + }, + FilterType::Peaking => { + if normalized == 0. || normalized == 1. { + self.constant_z_transform(1.); + return; + } else if q <= 0. { + self.constant_z_transform(a * a); + return; + } + }, + FilterType::AllPass => { + if normalized == 0. || normalized == 1. { + self.constant_z_transform(1.); + return; + } else if q <= 0. { + self.constant_z_transform(-1.); + return; + } + }, + FilterType::Notch => { + if normalized == 0. || normalized == 1. { + self.constant_z_transform(1.); + return; + } else if q <= 0. { + self.constant_z_transform(0.); + return; + } + }, + FilterType::BandPass => { + if normalized == 0. || normalized == 1. { + self.constant_z_transform(0.); + return; + } else if q <= 0. { + self.constant_z_transform(1.); + return; + } + }, + } + + let omega0 = 2. * PI * normalized; + let sin_omega = omega0.sin(); + let cos_omega = omega0.cos(); + let alpha_q = sin_omega / (2. * q); + let alpha_q_db = sin_omega / (2. * 10.0_f64.powf(q / 20.)); + let alpha_s = sin_omega / SQRT_2; + + // we predivide by a0 + let a0; + + match self.filter { + FilterType::LowPass => { + self.b0 = (1. - cos_omega) / 2.; + self.b1 = 1. - cos_omega; + self.b2 = self.b1 / 2.; + a0 = 1. + alpha_q_db; + self.a1 = -2. * cos_omega; + self.a2 = 1. - alpha_q_db; + }, + FilterType::HighPass => { + self.b0 = (1. + cos_omega) / 2.; + self.b1 = -(1. + cos_omega); + self.b2 = -self.b1 / 2.; + a0 = 1. + alpha_q_db; + self.a1 = -2. * cos_omega; + self.a2 = 1. - alpha_q_db; + }, + FilterType::BandPass => { + self.b0 = alpha_q; + self.b1 = 0.; + self.b2 = -alpha_q; + a0 = 1. + alpha_q; + self.a1 = -2. * cos_omega; + self.a2 = 1. - alpha_q; + }, + FilterType::Notch => { + self.b0 = 1.; + self.b1 = -2. * cos_omega; + self.b2 = 1.; + a0 = 1. + alpha_q; + self.a1 = -2. * cos_omega; + self.a2 = 1. - alpha_q; + }, + FilterType::AllPass => { + self.b0 = 1. - alpha_q; + self.b1 = -2. * cos_omega; + self.b2 = 1. + alpha_q; + a0 = 1. + alpha_q; + self.a1 = -2. * cos_omega; + self.a2 = 1. - alpha_q; + }, + FilterType::Peaking => { + self.b0 = 1. + alpha_q * a; + self.b1 = -2. * cos_omega; + self.b2 = 1. - alpha_q * a; + a0 = 1. + alpha_q / a; + self.a1 = -2. * cos_omega; + self.a2 = 1. - alpha_q / a; + }, + FilterType::LowShelf => { + let alpha_rt_a = 2. * alpha_s * a.sqrt(); + self.b0 = a * ((a + 1.) - (a - 1.) * cos_omega + alpha_rt_a); + self.b1 = 2. * a * ((a - 1.) - (a + 1.) * cos_omega); + self.b2 = a * ((a + 1.) - (a - 1.) * cos_omega - alpha_rt_a); + a0 = (a + 1.) + (a - 1.) * cos_omega + alpha_rt_a; + self.a1 = -2. * ((a - 1.) + (a + 1.) * cos_omega); + self.a2 = (a + 1.) + (a - 1.) * cos_omega - alpha_rt_a; + }, + FilterType::HighShelf => { + let alpha_rt_a = 2. * alpha_s * a.sqrt(); + self.b0 = a * ((a + 1.) + (a - 1.) * cos_omega + alpha_rt_a); + self.b1 = -2. * a * ((a - 1.) + (a + 1.) * cos_omega); + self.b2 = a * ((a + 1.) + (a - 1.) * cos_omega - alpha_rt_a); + a0 = (a + 1.) - (a - 1.) * cos_omega + alpha_rt_a; + self.a1 = 2. * ((a - 1.) - (a + 1.) * cos_omega); + self.a2 = (a + 1.) - (a - 1.) * cos_omega - alpha_rt_a; + }, + } + self.b0 /= a0; + self.b1 /= a0; + self.b2 /= a0; + self.a1 /= a0; + self.a2 /= a0; + } +} + +impl AudioNodeEngine for BiquadFilterNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::BiquadFilterNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + self.state + .resize(inputs.blocks[0].chan_count() as usize, Default::default()); + self.update_parameters(info, Tick(0)); + + // XXXManishearth this node has tail time, so even if the block is silence + // we must still compute things on it. However, it is possible to become + // a dumb passthrough as long as we reach a quiescent state + // + // see https://dxr.mozilla.org/mozilla-central/rev/87a95e1b7ec691bef7b938e722fe1b01cce68664/dom/media/webaudio/blink/Biquad.cpp#81-91 + + let repeat_or_silence = inputs.blocks[0].is_silence() || inputs.blocks[0].is_repeat(); + + if repeat_or_silence && !self.state.iter().all(|s| *s == self.state[0]) { + // In case our input is repeat/silence but our states are not identical, we must + // explicitly duplicate, since mutate_with will otherwise only operate + // on the first channel, ignoring the states of the later ones + inputs.blocks[0].explicit_repeat(); + } else { + // In case the states are identical, just make any silence explicit, + // since mutate_with can't handle silent blocks + inputs.blocks[0].explicit_silence(); + } + + { + let mut iter = inputs.blocks[0].iter(); + while let Some(mut frame) = iter.next() { + self.update_parameters(info, frame.tick()); + frame.mutate_with(|sample, chan| { + let state = &mut self.state[chan as usize]; + let x0 = *sample as f64; + let y0 = self.b0 * x0 + self.b1 * state.x1 + self.b2 * state.x2 - + self.a1 * state.y1 - + self.a2 * state.y2; + *sample = y0 as f32; + state.update(x0, y0); + }); + } + } + + if inputs.blocks[0].is_repeat() { + let state = self.state[0]; + self.state.iter_mut().for_each(|s| *s = state); + } + + inputs + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::Frequency => &mut self.frequency, + ParamType::Detune => &mut self.detune, + ParamType::Q => &mut self.q, + ParamType::Gain => &mut self.gain, + _ => panic!("Unknown param {:?} for BiquadFilterNode", id), + } + } + + fn message_specific(&mut self, message: AudioNodeMessage, sample_rate: f32) { + if let AudioNodeMessage::BiquadFilterNode(m) = message { + match m { + BiquadFilterNodeMessage::SetFilterType(f) => { + self.filter = f; + self.update_coefficients(sample_rate); + }, + } + } + } +} diff --git a/components/media/audio/block.rs b/components/media/audio/block.rs new file mode 100644 index 00000000000..ca474ef1d0e --- /dev/null +++ b/components/media/audio/block.rs @@ -0,0 +1,663 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::f32::consts::SQRT_2; +use std::mem; +use std::ops::*; + +use byte_slice_cast::*; +use euclid::default::Vector3D; +use smallvec::{SmallVec, smallvec}; + +use crate::graph::{PortIndex, PortKind}; +use crate::node::ChannelInterpretation; + +// defined by spec +// https://webaudio.github.io/web-audio-api/#render-quantum +pub const FRAMES_PER_BLOCK: Tick = Tick(128); +pub const FRAMES_PER_BLOCK_USIZE: usize = FRAMES_PER_BLOCK.0 as usize; + +/// A tick, i.e. the time taken for a single frame +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)] +pub struct Tick(pub u64); + +/// A collection of blocks received as input by a node +/// or outputted by a node. +/// +/// This will usually be a single block. +/// +/// Some nodes have multiple inputs or outputs, which is +/// where this becomes useful. Source nodes have an input +/// of an empty chunk. +pub struct Chunk { + pub blocks: SmallVec<[Block; 1]>, +} + +impl Default for Chunk { + fn default() -> Self { + Chunk { + blocks: SmallVec::new(), + } + } +} + +impl Chunk { + pub fn is_empty(&self) -> bool { + self.blocks.is_empty() + } + + pub fn len(&self) -> usize { + self.blocks.len() + } + + pub fn explicit_silence() -> Self { + let mut block = Block::default(); + block.explicit_silence(); + let blocks = smallvec![block]; + Self { blocks } + } +} + +/// We render audio in blocks of size FRAMES_PER_BLOCK +/// +/// A single block may contain multiple channels +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct Block { + /// The number of channels in this block + channels: u8, + /// This is an optimization which means that the buffer is representing multiple channels with the + /// same content at once. Happens when audio is upmixed or when a source like + /// an oscillator node has multiple channel outputs + repeat: bool, + /// If this vector is empty, it is a shorthand for "silence" + /// It is possible to obtain an explicitly silent buffer via .explicit_silence() + /// + /// This must be of length channels * FRAMES_PER_BLOCK, unless `repeat` is true, + /// in which case it will be of length FRAMES_PER_BLOCK + buffer: Vec, +} + +impl Default for Block { + fn default() -> Self { + Block { + channels: 1, + repeat: false, + buffer: Vec::new(), + } + } +} + +impl Block { + /// Empty block with no channels, for pushing + /// new channels to. + /// + /// Must be used with push_chan + pub fn empty() -> Self { + Block { + channels: 0, + ..Default::default() + } + } + + pub fn for_channels_explicit(channels: u8) -> Self { + Block { + channels, + repeat: false, + buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize], + } + } + + /// This provides the entire buffer as a mutable slice of u8 + pub fn as_mut_byte_slice(&mut self) -> &mut [u8] { + self.data_mut().as_mut_byte_slice() + } + + pub fn for_vec(buffer: Vec) -> Self { + assert!(buffer.len() % FRAMES_PER_BLOCK_USIZE == 0); + Block { + channels: (buffer.len() / FRAMES_PER_BLOCK_USIZE) as u8, + repeat: false, + buffer, + } + } + + /// Zero-gain sum with another buffer + /// + /// Used after mixing multiple inputs to a single port + pub fn sum(mut self, mut other: Self) -> Self { + if self.is_silence() { + other + } else if other.is_silence() { + self + } else { + debug_assert_eq!(self.channels, other.channels); + if self.repeat ^ other.repeat { + self.explicit_repeat(); + other.explicit_repeat(); + } + debug_assert_eq!(self.buffer.len(), other.buffer.len()); + for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) { + *a += b + } + self + } + } + + /// If this is in "silence" mode without a buffer, allocate a silent buffer + pub fn explicit_silence(&mut self) { + if self.buffer.is_empty() { + self.buffer.resize(FRAMES_PER_BLOCK_USIZE, 0.); + self.repeat = true; + } + } + + /// This provides the entire buffer as a mutable slice of f32 + pub fn data_mut(&mut self) -> &mut [f32] { + self.explicit_silence(); + &mut self.buffer + } + + pub fn explicit_repeat(&mut self) { + if self.repeat { + debug_assert!(self.buffer.len() == FRAMES_PER_BLOCK_USIZE); + if self.channels > 1 { + let mut new = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE * self.channels as usize); + for _ in 0..self.channels { + new.extend(&self.buffer) + } + + self.buffer = new; + } + self.repeat = false; + } else if self.is_silence() { + self.buffer + .resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.); + } + } + + pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] { + self.explicit_repeat(); + let start = chan as usize * FRAMES_PER_BLOCK_USIZE; + &mut self.buffer[start..start + FRAMES_PER_BLOCK_USIZE] + } + + #[inline] + pub fn data_chan(&self, chan: u8) -> &[f32] { + debug_assert!( + !self.is_silence(), + "data_chan doesn't work with silent buffers" + ); + let offset = if self.repeat { + 0 + } else { + chan as usize * FRAMES_PER_BLOCK_USIZE + }; + &self.buffer[offset..offset + FRAMES_PER_BLOCK_USIZE] + } + + pub fn take(&mut self) -> Block { + let new = Block { + channels: self.channels, + ..Default::default() + }; + mem::replace(self, new) + } + + pub fn chan_count(&self) -> u8 { + self.channels + } + + pub fn iter(&mut self) -> FrameIterator<'_> { + FrameIterator::new(self) + } + + pub fn is_silence(&self) -> bool { + self.buffer.is_empty() + } + + pub fn is_repeat(&self) -> bool { + self.repeat + } + + pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 { + if self.is_silence() { + 0. + } else { + self.data_chan(chan)[frame] + } + } + + pub fn push_chan(&mut self, data: &[f32]) { + assert!(!self.repeat); + assert!(!self.is_silence() || self.channels == 0); + assert!(data.len() == FRAMES_PER_BLOCK_USIZE); + self.buffer.extend(data); + self.channels += 1; + } + + /// upmix/downmix the channels if necessary + /// + /// Currently only supports upmixing from 1 + pub fn mix(&mut self, channels: u8, interpretation: ChannelInterpretation) { + // If we're not changing the number of channels, we + // don't actually need to mix + if self.channels == channels { + return; + } + + // Silent buffers stay silent + if self.is_silence() { + self.channels = channels; + return; + } + + if interpretation == ChannelInterpretation::Discrete { + // discrete downmixes by truncation, upmixes by adding + // silent channels + + // If we're discrete, have a repeat, and are downmixing, + // just truncate by changing the channel value + if self.repeat && self.channels > channels { + self.channels = channels; + } else { + // otherwise resize the buffer, silent-filling when necessary + self.resize_silence(channels); + } + } else { + // For speakers, we have to do special things based on the + // interpretation of the channels for each kind of speakers + + // The layout of each speaker kind is: + // + // - Mono: [The mono channel] + // - Stereo: [L, R] + // - Quad: [L, R, SL, SR] + // - 5.1: [L, R, C, LFE, SL, SR] + + match (self.channels, channels) { + // Upmixing + // https://webaudio.github.io/web-audio-api/#UpMix-sub + + // mono + (1, 2) => { + // output.{L, R} = input + self.repeat(2); + }, + (1, 4) => { + // output.{L, R} = input + self.repeat(2); + // output.{SL, SR} = 0 + self.resize_silence(4); + }, + (1, 6) => { + let mut v = Vec::with_capacity(channels as usize * FRAMES_PER_BLOCK_USIZE); + // output.{L, R} = 0 + v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.); + // output.C = input + v.extend(&self.buffer); + self.buffer = v; + // output.{LFE, SL, SR} = 0 + self.resize_silence(6); + }, + + // stereo + (2, 4) | (2, 6) => { + // output.{L, R} = input.{L, R} + // (5.1) output.{C, LFE} = 0 + // output.{SL, SR} = 0 + self.resize_silence(channels); + }, + + // quad + (4, 6) => { + // we can avoid this and instead calculate offsets + // based off whether or not this is `repeat`, but + // a `repeat` quad block should be rare + self.explicit_repeat(); + + let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE); + // output.{L, R} = input.{L, R} + v.extend(&self.buffer[0..2 * FRAMES_PER_BLOCK_USIZE]); + // output.{C, LFE} = 0 + v.resize(4 * FRAMES_PER_BLOCK_USIZE, 0.); + // output.{SL, R} = input.{SL, SR} + v.extend(&self.buffer[2 * FRAMES_PER_BLOCK_USIZE..]); + self.buffer = v; + self.channels = channels; + }, + + // Downmixing + // https://webaudio.github.io/web-audio-api/#down-mix + + // mono + (2, 1) => { + let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output = 0.5 * (input.L + input.R); + let o = + 0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 1)); + v.push(o); + } + self.buffer = v; + self.channels = 1; + self.repeat = false; + }, + (4, 1) => { + let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output = 0.5 * (input.L + input.R + input.SL + input.SR); + let o = 0.25 * + (self.data_chan_frame(frame, 0) + + self.data_chan_frame(frame, 1) + + self.data_chan_frame(frame, 2) + + self.data_chan_frame(frame, 3)); + v.push(o); + } + self.buffer = v; + self.channels = 1; + self.repeat = false; + }, + (6, 1) => { + let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output = sqrt(0.5) * (input.L + input.R) + input.C + 0.5 * (input.SL + input.SR) + let o = + // sqrt(0.5) * (input.L + input.R) + SQRT_2 * (self.data_chan_frame(frame, 0) + + self.data_chan_frame(frame, 1)) + + // input.C + self.data_chan_frame(frame, 2) + + // (ignore LFE) + // + 0 * self.buffer[frame + 3 * FRAMES_PER_BLOCK_USIZE] + // 0.5 * (input.SL + input.SR) + 0.5 * (self.data_chan_frame(frame, 4) + + self.data_chan_frame(frame, 5)); + v.push(o); + } + self.buffer = v; + self.channels = 1; + self.repeat = false; + }, + + // stereo + (4, 2) => { + let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE); + v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output.L = 0.5 * (input.L + input.SL) + v[frame] = + 0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 2)); + // output.R = 0.5 * (input.R + input.SR) + v[frame + FRAMES_PER_BLOCK_USIZE] = + 0.5 * (self.data_chan_frame(frame, 1) + self.data_chan_frame(frame, 3)); + } + self.buffer = v; + self.channels = 2; + self.repeat = false; + }, + (6, 2) => { + let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE); + v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output.L = L + sqrt(0.5) * (input.C + input.SL) + v[frame] = self.data_chan_frame(frame, 0) + + SQRT_2 * + (self.data_chan_frame(frame, 2) + + self.data_chan_frame(frame, 4)); + // output.R = R + sqrt(0.5) * (input.C + input.SR) + v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) + + SQRT_2 * + (self.data_chan_frame(frame, 2) + + self.data_chan_frame(frame, 5)); + } + self.buffer = v; + self.channels = 2; + self.repeat = false; + }, + + // quad + (6, 4) => { + let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE); + v.resize(6 * FRAMES_PER_BLOCK_USIZE, 0.); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output.L = L + sqrt(0.5) * input.C + v[frame] = self.data_chan_frame(frame, 0) + + SQRT_2 * self.data_chan_frame(frame, 2); + // output.R = R + sqrt(0.5) * input.C + v[frame + FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 1) + + SQRT_2 * self.data_chan_frame(frame, 2); + // output.SL = input.SL + v[frame + 2 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 4); + // output.SR = input.SR + v[frame + 3 * FRAMES_PER_BLOCK_USIZE] = self.data_chan_frame(frame, 5); + } + self.buffer = v; + self.channels = 4; + self.repeat = false; + }, + + // If it's not a known kind of speaker configuration, treat as + // discrete + _ => { + self.mix(channels, ChannelInterpretation::Discrete); + }, + } + debug_assert!(self.channels == channels); + } + } + + /// Resize to add or remove channels, fill extra channels with silence + pub fn resize_silence(&mut self, channels: u8) { + self.explicit_repeat(); + self.buffer + .resize(FRAMES_PER_BLOCK_USIZE * channels as usize, 0.); + self.channels = channels; + } + + /// Take a single-channel block and repeat the + /// channel + pub fn repeat(&mut self, channels: u8) { + debug_assert!(self.channels == 1); + self.channels = channels; + if !self.is_silence() { + self.repeat = true; + } + } + + pub fn interleave(&mut self) -> Vec { + self.explicit_repeat(); + let mut vec = Vec::with_capacity(self.buffer.len()); + // FIXME this isn't too efficient + vec.resize(self.buffer.len(), 0.); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + let channels = self.channels as usize; + for chan in 0..channels { + vec[frame * channels + chan] = self.buffer[chan * FRAMES_PER_BLOCK_USIZE + frame] + } + } + vec + } + + pub fn is_empty(&self) -> bool { + self.buffer.is_empty() + } + + /// Get the position, forward, and up vectors for a given + /// AudioListener-produced block + pub fn listener_data(&self, frame: Tick) -> (Vector3D, Vector3D, Vector3D) { + let frame = frame.0 as usize; + ( + Vector3D::new( + self.data_chan_frame(frame, 0), + self.data_chan_frame(frame, 1), + self.data_chan_frame(frame, 2), + ), + Vector3D::new( + self.data_chan_frame(frame, 3), + self.data_chan_frame(frame, 4), + self.data_chan_frame(frame, 5), + ), + Vector3D::new( + self.data_chan_frame(frame, 6), + self.data_chan_frame(frame, 7), + self.data_chan_frame(frame, 8), + ), + ) + } +} + +/// An iterator over frames in a block +pub struct FrameIterator<'a> { + frame: Tick, + block: &'a mut Block, +} + +impl<'a> FrameIterator<'a> { + #[inline] + pub fn new(block: &'a mut Block) -> Self { + FrameIterator { + frame: Tick(0), + block, + } + } + + /// Advance the iterator + /// + /// We can't implement Iterator since it doesn't support + /// streaming iterators, but we can call `while let Some(frame) = iter.next()` + /// here + #[inline] + pub fn next<'b>(&'b mut self) -> Option> { + let curr = self.frame; + if curr < FRAMES_PER_BLOCK { + self.frame.advance(); + Some(FrameRef { + frame: curr, + block: self.block, + }) + } else { + None + } + } +} + +/// A reference to a frame +pub struct FrameRef<'a> { + frame: Tick, + block: &'a mut Block, +} + +impl<'a> FrameRef<'a> { + #[inline] + pub fn tick(&self) -> Tick { + self.frame + } + + /// Given a block and a function `f`, mutate the frame through all channels with `f` + /// + /// Use this when you plan to do the same operation for each channel. + /// (Helpers for the other cases will eventually exist) + /// + /// Block must not be silence + /// + /// The second parameter to f is the channel number, 0 in case of a repeat() + #[inline] + pub fn mutate_with(&mut self, mut f: F) + where + F: FnMut(&mut f32, u8), + { + debug_assert!( + !self.block.is_silence(), + "mutate_frame_with should not be called with a silenced block, \ + call .explicit_silence() if you wish to use this" + ); + if self.block.repeat { + f(&mut self.block.buffer[self.frame.0 as usize], 0) + } else { + for chan in 0..self.block.channels { + f( + &mut self.block.buffer + [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize], + chan, + ) + } + } + } +} + +// operator impls + +impl IndexMut> for Chunk { + fn index_mut(&mut self, i: PortIndex) -> &mut Block { + if let PortIndex::Port(i) = i { + &mut self.blocks[i as usize] + } else { + panic!("attempted to index chunk with param") + } + } +} + +impl Index> for Chunk { + type Output = Block; + fn index(&self, i: PortIndex) -> &Block { + if let PortIndex::Port(i) = i { + &self.blocks[i as usize] + } else { + panic!("attempted to index chunk with param") + } + } +} + +impl Add for Tick { + type Output = Tick; + fn add(self, other: Tick) -> Self { + self + other.0 + } +} + +impl AddAssign for Tick { + fn add_assign(&mut self, other: Tick) { + *self = *self + other + } +} + +impl Sub for Tick { + type Output = Tick; + fn sub(self, other: Tick) -> Self { + self - other.0 + } +} + +impl Add for Tick { + type Output = Tick; + fn add(self, other: u64) -> Self { + Tick(self.0 + other) + } +} + +impl Sub for Tick { + type Output = Tick; + fn sub(self, other: u64) -> Self { + Tick(self.0 - other) + } +} + +impl Div for Tick { + type Output = f64; + fn div(self, other: f64) -> f64 { + self.0 as f64 / other + } +} + +impl Tick { + pub const FRAMES_PER_BLOCK: Tick = FRAMES_PER_BLOCK; + const EPSILON: f64 = 1e-7; + pub fn from_time(time: f64, rate: f32) -> Tick { + Tick((time * rate as f64 - Tick::EPSILON).ceil() as u64) + } + + pub fn advance(&mut self) { + self.0 += 1; + } +} diff --git a/components/media/audio/buffer_source_node.rs b/components/media/audio/buffer_source_node.rs new file mode 100644 index 00000000000..6a24623a5e0 --- /dev/null +++ b/components/media/audio/buffer_source_node.rs @@ -0,0 +1,439 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::block::{Block, Chunk, FRAMES_PER_BLOCK, Tick}; +use crate::node::{ + AudioNodeEngine, AudioNodeType, AudioScheduledSourceNodeMessage, BlockInfo, ChannelInfo, + OnEndedCallback, ShouldPlay, +}; +use crate::param::{Param, ParamType}; + +/// Control messages directed to AudioBufferSourceNodes. +#[derive(Debug, Clone)] +pub enum AudioBufferSourceNodeMessage { + /// Set the data block holding the audio sample data to be played. + SetBuffer(Option), + /// Set loop parameter. + SetLoopEnabled(bool), + /// Set loop parameter. + SetLoopEnd(f64), + /// Set loop parameter. + SetLoopStart(f64), + /// Set start parameters (when, offset, duration). + SetStartParams(f64, Option, Option), +} + +/// This specifies options for constructing an AudioBufferSourceNode. +#[derive(Debug, Clone)] +pub struct AudioBufferSourceNodeOptions { + /// The audio asset to be played. + pub buffer: Option, + /// The initial value for the detune AudioParam. + pub detune: f32, + /// The initial value for the loop_enabled attribute. + pub loop_enabled: bool, + /// The initial value for the loop_end attribute. + pub loop_end: Option, + /// The initial value for the loop_start attribute. + pub loop_start: Option, + /// The initial value for the playback_rate AudioParam. + pub playback_rate: f32, +} + +impl Default for AudioBufferSourceNodeOptions { + fn default() -> Self { + AudioBufferSourceNodeOptions { + buffer: None, + detune: 0., + loop_enabled: false, + loop_end: None, + loop_start: None, + playback_rate: 1., + } + } +} + +/// AudioBufferSourceNode engine. +/// +#[derive(AudioScheduledSourceNode, AudioNodeCommon)] +#[allow(dead_code)] +pub(crate) struct AudioBufferSourceNode { + channel_info: ChannelInfo, + /// A data block holding the audio sample data to be played. + buffer: Option, + /// How many more buffer-frames to output. See buffer_pos for clarification. + buffer_duration: f64, + /// "Index" of the next buffer frame to play. "Index" is in quotes because + /// this variable maps to a playhead position (the offset in seconds can be + /// obtained by dividing by self.buffer.sample_rate), and therefore has + /// subsample accuracy; a fractional "index" means interpolation is needed. + buffer_pos: f64, + /// AudioParam to modulate the speed at which is rendered the audio stream. + detune: Param, + /// Whether we need to compute offsets from scratch. + initialized_pos: bool, + /// Indicates if the region of audio data designated by loopStart and loopEnd + /// should be played continuously in a loop. + loop_enabled: bool, + /// An playhead position where looping should end if the loop_enabled + /// attribute is true. + loop_end: Option, + /// An playhead position where looping should begin if the loop_enabled + /// attribute is true. + loop_start: Option, + /// The speed at which to render the audio stream. Can be negative if the + /// audio is to be played backwards. With a negative playback_rate, looping + /// jumps from loop_start to loop_end instead of the other way around. + playback_rate: Param, + /// Time at which the source should start playing. + start_at: Option, + /// Offset parameter passed to Start(). + start_offset: Option, + /// Duration parameter passed to Start(). + start_duration: Option, + /// The same as start_at, but with subsample accuracy. + /// FIXME: AudioScheduledSourceNode should use this as well. + start_when: f64, + /// Time at which the source should stop playing. + stop_at: Option, + /// The ended event callback. + pub onended_callback: Option, +} + +impl AudioBufferSourceNode { + pub fn new(options: AudioBufferSourceNodeOptions, channel_info: ChannelInfo) -> Self { + Self { + channel_info, + buffer: options.buffer, + buffer_pos: 0., + detune: Param::new_krate(options.detune), + initialized_pos: false, + loop_enabled: options.loop_enabled, + loop_end: options.loop_end, + loop_start: options.loop_start, + playback_rate: Param::new_krate(options.playback_rate), + buffer_duration: f64::INFINITY, + start_at: None, + start_offset: None, + start_duration: None, + start_when: 0., + stop_at: None, + onended_callback: None, + } + } + + pub fn handle_message(&mut self, message: AudioBufferSourceNodeMessage, _: f32) { + match message { + AudioBufferSourceNodeMessage::SetBuffer(buffer) => { + self.buffer = buffer; + }, + // XXX(collares): To fully support dynamically updating loop bounds, + // Must truncate self.buffer_pos if it is now outside the loop. + AudioBufferSourceNodeMessage::SetLoopEnabled(loop_enabled) => { + self.loop_enabled = loop_enabled + }, + AudioBufferSourceNodeMessage::SetLoopEnd(loop_end) => self.loop_end = Some(loop_end), + AudioBufferSourceNodeMessage::SetLoopStart(loop_start) => { + self.loop_start = Some(loop_start) + }, + AudioBufferSourceNodeMessage::SetStartParams(when, offset, duration) => { + self.start_when = when; + self.start_offset = offset; + self.start_duration = duration; + }, + } + } +} + +impl AudioNodeEngine for AudioBufferSourceNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::AudioBufferSourceNode + } + + fn input_count(&self) -> u32 { + 0 + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.is_empty()); + + if self.buffer.is_none() { + inputs.blocks.push(Default::default()); + return inputs; + } + + let (start_at, stop_at) = match self.should_play_at(info.frame) { + ShouldPlay::No => { + inputs.blocks.push(Default::default()); + return inputs; + }, + ShouldPlay::Between(start, end) => (start.0 as usize, end.0 as usize), + }; + + let buffer = self.buffer.as_ref().unwrap(); + + let (mut actual_loop_start, mut actual_loop_end) = (0., buffer.len() as f64); + if self.loop_enabled { + let loop_start = self.loop_start.unwrap_or(0.); + let loop_end = self.loop_end.unwrap_or(0.); + + if loop_start >= 0. && loop_end > loop_start { + actual_loop_start = loop_start * (buffer.sample_rate as f64); + actual_loop_end = loop_end * (buffer.sample_rate as f64); + } + } + + // https://webaudio.github.io/web-audio-api/#computedplaybackrate + self.playback_rate.update(info, Tick(0)); + self.detune.update(info, Tick(0)); + // computed_playback_rate can be negative or zero. + let computed_playback_rate = + self.playback_rate.value() as f64 * (2.0_f64).powf(self.detune.value() as f64 / 1200.); + let forward = computed_playback_rate >= 0.; + + if !self.initialized_pos { + self.initialized_pos = true; + + // Apply the offset and duration parameters passed to start. We handle + // this here because the buffer may be set after Start() gets called, so + // this might be the first time we know the buffer's sample rate. + if let Some(start_offset) = self.start_offset { + self.buffer_pos = start_offset * (buffer.sample_rate as f64); + if self.buffer_pos < 0. { + self.buffer_pos = 0. + } else if self.buffer_pos > buffer.len() as f64 { + self.buffer_pos = buffer.len() as f64; + } + } + + if self.loop_enabled { + if forward && self.buffer_pos >= actual_loop_end { + self.buffer_pos = actual_loop_start; + } + // https://github.com/WebAudio/web-audio-api/issues/2031 + if !forward && self.buffer_pos < actual_loop_start { + self.buffer_pos = actual_loop_end; + } + } + + if let Some(start_duration) = self.start_duration { + self.buffer_duration = start_duration * (buffer.sample_rate as f64); + } + + // start_when can be subsample accurate. Correct buffer_pos. + // + // XXX(collares): What happens to "start_when" if the buffer gets + // set after Start()? + // XXX(collares): Need a better way to distingush between Start() + // being called with "when" in the past (in which case "when" must + // be ignored) and Start() being called with "when" in the future. + // This can now make a difference if "when" shouldn't be ignored + // but falls after the last frame of the previous quantum. + if self.start_when > info.time - 1. / info.sample_rate as f64 { + let first_time = info.time + start_at as f64 / info.sample_rate as f64; + if self.start_when <= first_time { + let subsample_offset = (first_time - self.start_when) * + (buffer.sample_rate as f64) * + computed_playback_rate; + self.buffer_pos += subsample_offset; + self.buffer_duration -= subsample_offset.abs(); + } + } + } + + let mut buffer_offset_per_tick = + computed_playback_rate * (buffer.sample_rate as f64 / info.sample_rate as f64); + + // WebAudio §1.9.5: "Setting the loop attribute to true causes playback of + // the region of the buffer defined by the endpoints loopStart and loopEnd + // to continue indefinitely, once any part of the looped region has been + // played. While loop remains true, looped playback will continue until one + // of the following occurs: + // * stop() is called, + // * the scheduled stop time has been reached, + // * the duration has been exceeded, if start() was called with a duration value." + // Even with extreme playback rates we must stay inside the loop body, so wrap + // the per-tick delta instead of bailing. + if self.loop_enabled && actual_loop_end > actual_loop_start { + let loop_length = actual_loop_end - actual_loop_start; + if loop_length > 0. { + let step = buffer_offset_per_tick.abs(); + if step >= loop_length { + let mut wrapped = step.rem_euclid(loop_length); + if wrapped == 0. { + wrapped = loop_length; + } + buffer_offset_per_tick = wrapped.copysign(buffer_offset_per_tick); + } + } + } + + // We will output at most this many frames (fewer if we run out of data). + let frames_to_output = stop_at - start_at; + + // Fast path for the case where we can just copy FRAMES_PER_BLOCK + // frames straight from the buffer. + if frames_to_output == FRAMES_PER_BLOCK.0 as usize && + forward && + buffer_offset_per_tick == 1. && + self.buffer_pos.trunc() == self.buffer_pos && + self.buffer_pos + (FRAMES_PER_BLOCK.0 as f64) <= actual_loop_end && + FRAMES_PER_BLOCK.0 as f64 <= self.buffer_duration + { + let mut block = Block::empty(); + let pos = self.buffer_pos as usize; + + for chan in 0..buffer.chans() { + block.push_chan(&buffer.buffers[chan as usize][pos..(pos + frames_to_output)]); + } + + inputs.blocks.push(block); + self.buffer_pos += FRAMES_PER_BLOCK.0 as f64; + self.buffer_duration -= FRAMES_PER_BLOCK.0 as f64; + } else { + // Slow path, with interpolation. + let mut block = Block::default(); + block.repeat(buffer.chans()); + block.explicit_repeat(); + + debug_assert!(buffer.chans() > 0); + + for chan in 0..buffer.chans() { + let data = block.data_chan_mut(chan); + let (_, data) = data.split_at_mut(start_at); + let (data, _) = data.split_at_mut(frames_to_output); + + let mut pos = self.buffer_pos; + let mut duration = self.buffer_duration; + + for sample in data { + if duration <= 0. { + break; + } + + if self.loop_enabled { + if forward && pos >= actual_loop_end { + pos -= actual_loop_end - actual_loop_start; + } else if !forward && pos < actual_loop_start { + pos += actual_loop_end - actual_loop_start; + } + } else if pos < 0. || pos >= buffer.len() as f64 { + break; + } + + *sample = buffer.interpolate(chan, pos); + pos += buffer_offset_per_tick; + duration -= buffer_offset_per_tick.abs(); + } + + // This is the last channel, update parameters. + if chan == buffer.chans() - 1 { + self.buffer_pos = pos; + self.buffer_duration = duration; + } + } + + inputs.blocks.push(block); + } + + if !self.loop_enabled && (self.buffer_pos < 0. || self.buffer_pos >= buffer.len() as f64) || + self.buffer_duration <= 0. + { + self.maybe_trigger_onended_callback(); + } + + inputs + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::PlaybackRate => &mut self.playback_rate, + ParamType::Detune => &mut self.detune, + _ => panic!("Unknown param {:?} for AudioBufferSourceNode", id), + } + } + + make_message_handler!( + AudioBufferSourceNode: handle_message, + AudioScheduledSourceNode: handle_source_node_message + ); +} + +#[derive(Debug, Clone)] +pub struct AudioBuffer { + /// Invariant: all buffers must be of the same length + pub buffers: Vec>, + pub sample_rate: f32, +} + +impl AudioBuffer { + pub fn new(chan: u8, len: usize, sample_rate: f32) -> Self { + assert!(chan > 0); + let mut buffers = Vec::with_capacity(chan as usize); + let single = vec![0.; len]; + buffers.resize(chan as usize, single); + AudioBuffer { + buffers, + sample_rate, + } + } + + pub fn from_buffers(buffers: Vec>, sample_rate: f32) -> Self { + for buf in &buffers { + assert_eq!(buf.len(), buffers[0].len()) + } + + Self { + buffers, + sample_rate, + } + } + + pub fn from_buffer(buffer: Vec, sample_rate: f32) -> Self { + AudioBuffer::from_buffers(vec![buffer], sample_rate) + } + + pub fn len(&self) -> usize { + self.buffers[0].len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn chans(&self) -> u8 { + self.buffers.len() as u8 + } + + // XXX(collares): There are better fast interpolation algorithms. + // Firefox uses (via Speex's resampler) the algorithm described in + // https://ccrma.stanford.edu/~jos/resample/resample.pdf + // There are Rust bindings: https://github.com/rust-av/speexdsp-rs + pub fn interpolate(&self, chan: u8, pos: f64) -> f32 { + debug_assert!(pos >= 0. && pos < self.len() as f64); + + let prev = pos.floor() as usize; + let offset = pos - pos.floor(); + match self.buffers[chan as usize].get(prev + 1) { + Some(next_sample) => { + ((1. - offset) * (self.buffers[chan as usize][prev] as f64) + + offset * (*next_sample as f64)) as f32 + }, + _ => { + // linear extrapolation of two prev samples if there are two + if prev > 0 { + ((1. + offset) * (self.buffers[chan as usize][prev] as f64) - + offset * (self.buffers[chan as usize][prev - 1] as f64)) + as f32 + } else { + self.buffers[chan as usize][prev] + } + }, + } + } + + pub fn data_chan_mut(&mut self, chan: u8) -> &mut [f32] { + &mut self.buffers[chan as usize] + } +} diff --git a/components/media/audio/channel_node.rs b/components/media/audio/channel_node.rs new file mode 100644 index 00000000000..36cffd20977 --- /dev/null +++ b/components/media/audio/channel_node.rs @@ -0,0 +1,119 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE}; +use crate::node::{ + AudioNodeEngine, AudioNodeType, BlockInfo, ChannelCountMode, ChannelInfo, ChannelInterpretation, +}; + +#[derive(Copy, Clone, Debug)] +pub struct ChannelNodeOptions { + pub channels: u8, +} + +#[derive(AudioNodeCommon)] +pub(crate) struct ChannelMergerNode { + channel_info: ChannelInfo, + channels: u8, +} + +impl ChannelMergerNode { + pub fn new(params: ChannelNodeOptions, channel_info: ChannelInfo) -> Self { + ChannelMergerNode { + channel_info, + channels: params.channels, + } + } +} + +impl AudioNodeEngine for ChannelMergerNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::ChannelMergerNode + } + + fn process(&mut self, mut inputs: Chunk, _: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == self.channels as usize); + + let mut block = Block::default(); + block.repeat(self.channels); + block.explicit_repeat(); + + for (i, channel) in block + .data_mut() + .chunks_mut(FRAMES_PER_BLOCK_USIZE) + .enumerate() + { + channel.copy_from_slice(inputs.blocks[i].data_mut()) + } + + inputs.blocks.clear(); + inputs.blocks.push(block); + inputs + } + + fn input_count(&self) -> u32 { + self.channels as u32 + } + + fn set_channel_count_mode(&mut self, _: ChannelCountMode) { + panic!("channel merger nodes cannot have their mode changed"); + } + + fn set_channel_count(&mut self, _: u8) { + panic!("channel merger nodes cannot have their channel count changed"); + } +} + +#[derive(AudioNodeCommon)] +pub(crate) struct ChannelSplitterNode { + channel_info: ChannelInfo, +} + +impl ChannelSplitterNode { + pub fn new(channel_info: ChannelInfo) -> Self { + ChannelSplitterNode { channel_info } + } +} + +impl AudioNodeEngine for ChannelSplitterNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::ChannelSplitterNode + } + + fn process(&mut self, mut inputs: Chunk, _: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + let original = inputs.blocks.pop().unwrap(); + + if original.is_silence() { + inputs + .blocks + .resize(original.chan_count() as usize, Block::default()) + } else { + for chan in 0..original.chan_count() { + let mut block = Block::empty(); + block.push_chan(original.data_chan(chan)); + inputs.blocks.push(block); + } + } + + inputs + } + + fn output_count(&self) -> u32 { + self.channel_count() as u32 + } + + fn set_channel_count_mode(&mut self, _: ChannelCountMode) { + panic!("channel splitter nodes cannot have their mode changed"); + } + + fn set_channel_interpretation(&mut self, _: ChannelInterpretation) { + panic!("channel splitter nodes cannot have their channel interpretation changed"); + } + + fn set_channel_count(&mut self, _: u8) { + panic!("channel splitter nodes cannot have their channel count changed"); + } +} diff --git a/components/media/audio/constant_source_node.rs b/components/media/audio/constant_source_node.rs new file mode 100644 index 00000000000..0407e284fb7 --- /dev/null +++ b/components/media/audio/constant_source_node.rs @@ -0,0 +1,96 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::block::{Chunk, Tick}; +use crate::node::{ + AudioNodeEngine, AudioNodeType, AudioScheduledSourceNodeMessage, BlockInfo, ChannelInfo, + OnEndedCallback, ShouldPlay, +}; +use crate::param::{Param, ParamType}; + +#[derive(Copy, Clone, Debug)] +pub struct ConstantSourceNodeOptions { + pub offset: f32, +} + +impl Default for ConstantSourceNodeOptions { + fn default() -> Self { + ConstantSourceNodeOptions { offset: 1. } + } +} + +#[derive(AudioScheduledSourceNode, AudioNodeCommon)] +pub(crate) struct ConstantSourceNode { + channel_info: ChannelInfo, + offset: Param, + start_at: Option, + stop_at: Option, + onended_callback: Option, +} + +impl ConstantSourceNode { + pub fn new(options: ConstantSourceNodeOptions, channel_info: ChannelInfo) -> Self { + Self { + channel_info, + offset: Param::new(options.offset), + start_at: None, + stop_at: None, + onended_callback: None, + } + } + + pub fn update_parameters(&mut self, info: &BlockInfo, tick: Tick) -> bool { + self.offset.update(info, tick) + } +} + +impl AudioNodeEngine for ConstantSourceNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::ConstantSourceNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.is_empty()); + + inputs.blocks.push(Default::default()); + + let (start_at, stop_at) = match self.should_play_at(info.frame) { + ShouldPlay::No => { + return inputs; + }, + ShouldPlay::Between(start, end) => (start, end), + }; + + { + inputs.blocks[0].explicit_silence(); + + let mut iter = inputs.blocks[0].iter(); + let mut offset = self.offset.value(); + while let Some(mut frame) = iter.next() { + let tick = frame.tick(); + if tick < start_at { + continue; + } else if tick > stop_at { + break; + } + if self.update_parameters(info, frame.tick()) { + offset = self.offset.value(); + } + frame.mutate_with(|sample, _| *sample = offset); + } + } + inputs + } + fn input_count(&self) -> u32 { + 0 + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::Offset => &mut self.offset, + _ => panic!("Unknown param {:?} for the offset", id), + } + } + make_message_handler!(AudioScheduledSourceNode: handle_source_node_message); +} diff --git a/components/media/audio/context.rs b/components/media/audio/context.rs new file mode 100644 index 00000000000..c283403c696 --- /dev/null +++ b/components/media/audio/context.rs @@ -0,0 +1,362 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cell::Cell; +use std::sync::mpsc::{self, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread::Builder; + +use servo_media_traits::{BackendMsg, ClientContextId, MediaInstance, MediaInstanceError}; + +use crate::AudioBackend; +use crate::decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecoderOptions}; +use crate::graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; +use crate::node::{AudioNodeInit, AudioNodeMessage, ChannelInfo}; +use crate::render_thread::{AudioRenderThread, AudioRenderThreadMsg, SinkEosCallback}; +use crate::sink::AudioSinkError; + +/// Describes the state of the audio context on the control thread. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ProcessingState { + /// The audio context is suspended (context time is not proceeding, + /// audio hardware may be powered down/released). + Suspended, + /// Audio is being processed. + Running, + /// The audio context has been released, and can no longer be used + /// to process audio. + Closed, +} + +pub type StateChangeResult = Option<()>; + +/// Identify the type of playback, which affects tradeoffs between audio output +/// and power consumption. +#[derive(Copy, Clone)] +pub enum LatencyCategory { + /// Balance audio output latency and power consumption. + Balanced, + /// Provide the lowest audio output latency possible without glitching. + Interactive, + /// Prioritize sustained playback without interruption over audio output latency. + /// Lowest power consumption. + Playback, +} + +/// User-specified options for a real time audio context. +#[derive(Copy, Clone)] +pub struct RealTimeAudioContextOptions { + /// Number of samples that will play in one second, measured in Hz. + pub sample_rate: f32, + /// Type of playback. + pub latency_hint: LatencyCategory, +} + +impl Default for RealTimeAudioContextOptions { + fn default() -> Self { + Self { + sample_rate: 44100., + latency_hint: LatencyCategory::Interactive, + } + } +} + +/// User-specified options for an offline audio context. +#[derive(Copy, Clone)] +pub struct OfflineAudioContextOptions { + /// The number of channels for this offline audio context. + pub channels: u8, + /// The length of the rendered audio buffer in sample-frames. + pub length: usize, + /// Number of samples that will be rendered in one second, measured in Hz. + pub sample_rate: f32, +} + +impl Default for OfflineAudioContextOptions { + fn default() -> Self { + Self { + channels: 1, + length: 0, + sample_rate: 44100., + } + } +} + +impl From for AudioContextOptions { + fn from(options: RealTimeAudioContextOptions) -> Self { + AudioContextOptions::RealTimeAudioContext(options) + } +} + +impl From for AudioContextOptions { + fn from(options: OfflineAudioContextOptions) -> Self { + AudioContextOptions::OfflineAudioContext(options) + } +} + +/// User-specified options for a real time or offline audio context. +#[derive(Copy, Clone)] +pub enum AudioContextOptions { + RealTimeAudioContext(RealTimeAudioContextOptions), + OfflineAudioContext(OfflineAudioContextOptions), +} + +impl Default for AudioContextOptions { + fn default() -> Self { + AudioContextOptions::RealTimeAudioContext(Default::default()) + } +} + +/// Representation of an audio context on the control thread. +pub struct AudioContext { + /// Media instance ID. + id: usize, + /// Client context ID. + client_context_id: ClientContextId, + /// Owner backend communication channel. + backend_chan: Arc>>, + /// Rendering thread communication channel. + sender: Sender, + /// State of the audio context on the control thread. + state: Cell, + /// Number of samples that will be played in one second. + sample_rate: f32, + /// The identifier of an AudioDestinationNode with a single input + /// representing the final destination for all audio. + dest_node: NodeId, + listener: NodeId, + make_decoder: Arc Box + Sync + Send>, +} + +#[derive(Debug)] +pub struct AudioContextError; + +impl std::fmt::Display for AudioContextError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "AudioContextError") + } +} + +impl std::error::Error for AudioContextError {} + +impl AudioContext { + /// Constructs a new audio context. + pub fn new( + id: usize, + client_context_id: &ClientContextId, + backend_chan: Arc>>, + options: AudioContextOptions, + ) -> Result { + let (sample_rate, channels) = match options { + AudioContextOptions::RealTimeAudioContext(ref options) => (options.sample_rate, 2), + AudioContextOptions::OfflineAudioContext(ref options) => { + (options.sample_rate, options.channels) + }, + }; + + let (sender, receiver) = mpsc::channel(); + let sender_ = sender.clone(); + let graph = AudioGraph::new(channels); + let dest_node = graph.dest_id(); + let listener = graph.listener_id(); + + let (init_sender, init_receiver) = mpsc::channel(); + Builder::new() + .name("AudioRenderThread".to_owned()) + .spawn(move || { + AudioRenderThread::start::( + receiver, + sender_, + sample_rate, + graph, + options, + init_sender, + ) + }) + .expect("Failed to spawn AudioRenderThread"); + + init_receiver + .recv() + .expect("Failed to receive result from AudioRenderThread")?; + Ok(Self { + id, + client_context_id: *client_context_id, + backend_chan, + sender, + state: Cell::new(ProcessingState::Suspended), + sample_rate, + dest_node, + listener, + make_decoder: Arc::new(|| B::make_decoder()), + }) + } + + pub fn state(&self) -> ProcessingState { + self.state.get() + } + + pub fn dest_node(&self) -> NodeId { + self.dest_node + } + + pub fn listener(&self) -> NodeId { + self.listener + } + + pub fn current_time(&self) -> f64 { + let (tx, rx) = mpsc::channel(); + let _ = self.sender.send(AudioRenderThreadMsg::GetCurrentTime(tx)); + rx.recv().unwrap() + } + + pub fn create_node(&self, node_type: AudioNodeInit, ch: ChannelInfo) -> Option { + let (tx, rx) = mpsc::channel(); + let _ = self + .sender + .send(AudioRenderThreadMsg::CreateNode(node_type, tx, ch)); + rx.recv().ok() + } + + // Resume audio processing. + make_state_change!(resume, Running, Resume); + + // Suspend audio processing. + make_state_change!(suspend, Suspended, Suspend); + + // Stop audio processing and close render thread. + make_state_change!(close, Closed, Close); + + pub fn message_node(&self, id: NodeId, msg: AudioNodeMessage) { + let _ = self.sender.send(AudioRenderThreadMsg::MessageNode(id, msg)); + } + + pub fn connect_ports(&self, from: PortId, to: PortId) { + let _ = self + .sender + .send(AudioRenderThreadMsg::ConnectPorts(from, to)); + } + + pub fn disconnect_all_from(&self, node: NodeId) { + let _ = self + .sender + .send(AudioRenderThreadMsg::DisconnectAllFrom(node)); + } + + /// Disconnect all outgoing connections from a node's output + /// + /// > + pub fn disconnect_output(&self, out: PortId) { + let _ = self + .sender + .send(AudioRenderThreadMsg::DisconnectOutput(out)); + } + + /// Disconnect connections from a node to another node + /// + /// + pub fn disconnect_between(&self, from: NodeId, to: NodeId) { + let _ = self + .sender + .send(AudioRenderThreadMsg::DisconnectBetween(from, to)); + } + + /// Disconnect connections from a node to another node's input + /// + /// + pub fn disconnect_to(&self, from: NodeId, to: PortId) { + let _ = self + .sender + .send(AudioRenderThreadMsg::DisconnectTo(from, to)); + } + + /// Disconnect all outgoing connections from a node's output to another node + /// + /// + pub fn disconnect_output_between(&self, out: PortId, to: NodeId) { + let _ = self + .sender + .send(AudioRenderThreadMsg::DisconnectOutputBetween(out, to)); + } + + /// Disconnect all outgoing connections from a node's output to another node's input + /// + /// + pub fn disconnect_output_between_to(&self, out: PortId, inp: PortId) { + let _ = self + .sender + .send(AudioRenderThreadMsg::DisconnectOutputBetweenTo(out, inp)); + } + + /// Asynchronously decodes the audio file data contained in the given + /// buffer. + pub fn decode_audio_data(&self, data: Vec, callbacks: AudioDecoderCallbacks) { + let options = AudioDecoderOptions { + sample_rate: self.sample_rate, + }; + let make_decoder = self.make_decoder.clone(); + Builder::new() + .name("AudioDecoder".to_owned()) + .spawn(move || { + let audio_decoder = make_decoder(); + + audio_decoder.decode(data, callbacks, Some(options)); + }) + .unwrap(); + } + + pub fn set_eos_callback(&self, callback: SinkEosCallback) { + let _ = self + .sender + .send(AudioRenderThreadMsg::SetSinkEosCallback(callback)); + } + + fn set_mute(&self, val: bool) { + let _ = self.sender.send(AudioRenderThreadMsg::SetMute(val)); + } +} + +impl Drop for AudioContext { + fn drop(&mut self) { + let (tx, _) = mpsc::channel(); + let _ = self.sender.send(AudioRenderThreadMsg::Close(tx)); + + // Ask the backend to unregister this instance and wait for ACK + let (tx_ack, rx_ack) = mpsc::channel(); + let _ = self + .backend_chan + .lock() + .unwrap() + .send(BackendMsg::Shutdown { + context: self.client_context_id, + id: self.id, + tx_ack, + }); + let _ = rx_ack.recv(); + } +} + +impl MediaInstance for AudioContext { + fn get_id(&self) -> usize { + self.id + } + + fn mute(&self, val: bool) -> Result<(), MediaInstanceError> { + self.set_mute(val); + Ok(()) + } + + fn suspend(&self) -> Result<(), MediaInstanceError> { + let (tx, _) = mpsc::channel(); + self.sender + .send(AudioRenderThreadMsg::Suspend(tx)) + .map_err(|_| MediaInstanceError) + } + + fn resume(&self) -> Result<(), MediaInstanceError> { + let (tx, _) = mpsc::channel(); + self.sender + .send(AudioRenderThreadMsg::Resume(tx)) + .map_err(|_| MediaInstanceError) + } +} diff --git a/components/media/audio/decoder.rs b/components/media/audio/decoder.rs new file mode 100644 index 00000000000..d30124ce467 --- /dev/null +++ b/components/media/audio/decoder.rs @@ -0,0 +1,128 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::Mutex; + +#[derive(Debug, PartialEq)] +pub enum AudioDecoderError { + /// Backend specific error. + Backend(String), + /// Could not read the audio buffer content. + BufferReadFailed, + /// The media trying to be decoded has an invalid format. + InvalidMediaFormat, + /// An invalid sample was found while decoding the audio. + InvalidSample, + /// Could not move to a different state. + StateChangeFailed, +} + +type AudioDecoderEosCallback = Box; +type AudioDecoderErrorCallback = Box; +type AudioDecoderProgressCallback = Box>, u32) + Send + Sync + 'static>; +type AudioDecoderReadyCallback = Box; + +pub struct AudioDecoderCallbacks { + pub eos: Mutex>, + pub error: Mutex>, + pub progress: Option, + pub ready: Mutex>, +} + +impl AudioDecoderCallbacks { + pub fn eos(&self) { + if let Some(callback) = self.eos.lock().unwrap().take() { + callback(); + } + } + + pub fn error(&self, error: AudioDecoderError) { + if let Some(callback) = self.error.lock().unwrap().take() { + callback(error); + } + } + + pub fn progress(&self, buffer: Box>, channel: u32) { + if let Some(callback) = self.progress.as_ref() { + callback(buffer, channel); + } + } + + pub fn ready(&self, channels: u32) { + if let Some(callback) = self.ready.lock().unwrap().take() { + callback(channels); + } + } +} + +#[derive(Default)] +pub struct AudioDecoderCallbacksBuilder { + eos: Option, + error: Option, + progress: Option, + ready: Option, +} + +impl AudioDecoderCallbacksBuilder { + pub fn eos(self, eos: F) -> Self { + Self { + eos: Some(Box::new(eos)), + ..self + } + } + + pub fn error(self, error: F) -> Self { + Self { + error: Some(Box::new(error)), + ..self + } + } + + pub fn progress>, u32) + Send + Sync + 'static>( + self, + progress: F, + ) -> Self { + Self { + progress: Some(Box::new(progress)), + ..self + } + } + + pub fn ready(self, ready: F) -> Self { + Self { + ready: Some(Box::new(ready)), + ..self + } + } + + pub fn build(self) -> AudioDecoderCallbacks { + AudioDecoderCallbacks { + eos: Mutex::new(self.eos), + error: Mutex::new(self.error), + progress: self.progress, + ready: Mutex::new(self.ready), + } + } +} + +pub struct AudioDecoderOptions { + pub sample_rate: f32, +} + +impl Default for AudioDecoderOptions { + fn default() -> Self { + AudioDecoderOptions { + sample_rate: 44100., + } + } +} + +pub trait AudioDecoder { + fn decode( + &self, + data: Vec, + callbacks: AudioDecoderCallbacks, + options: Option, + ); +} diff --git a/components/media/audio/destination_node.rs b/components/media/audio/destination_node.rs new file mode 100644 index 00000000000..79af4bd925d --- /dev/null +++ b/components/media/audio/destination_node.rs @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::block::Chunk; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelCountMode, ChannelInfo}; + +#[derive(AudioNodeCommon)] +pub(crate) struct DestinationNode { + channel_info: ChannelInfo, + chunk: Option, +} + +impl DestinationNode { + pub fn new(channel_count: u8) -> Self { + DestinationNode { + channel_info: ChannelInfo { + mode: ChannelCountMode::Explicit, + count: channel_count, + ..Default::default() + }, + chunk: None, + } + } +} + +impl AudioNodeEngine for DestinationNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::DestinationNode + } + + fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk { + self.chunk = Some(inputs); + Chunk::default() + } + + fn destination_data(&mut self) -> Option { + self.chunk.take() + } + + fn output_count(&self) -> u32 { + 0 + } +} diff --git a/components/media/audio/gain_node.rs b/components/media/audio/gain_node.rs new file mode 100644 index 00000000000..5d9d3313252 --- /dev/null +++ b/components/media/audio/gain_node.rs @@ -0,0 +1,71 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::block::{Chunk, Tick}; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; +use crate::param::{Param, ParamType}; + +#[derive(Copy, Clone, Debug)] +pub struct GainNodeOptions { + pub gain: f32, +} + +impl Default for GainNodeOptions { + fn default() -> Self { + GainNodeOptions { gain: 1. } + } +} + +#[derive(AudioNodeCommon)] +pub(crate) struct GainNode { + channel_info: ChannelInfo, + gain: Param, +} + +impl GainNode { + pub fn new(options: GainNodeOptions, channel_info: ChannelInfo) -> Self { + Self { + channel_info, + gain: Param::new(options.gain), + } + } + + pub fn update_parameters(&mut self, info: &BlockInfo, tick: Tick) -> bool { + self.gain.update(info, tick) + } +} + +impl AudioNodeEngine for GainNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::GainNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + if inputs.blocks[0].is_silence() { + return inputs; + } + + { + let mut iter = inputs.blocks[0].iter(); + let mut gain = self.gain.value(); + + while let Some(mut frame) = iter.next() { + if self.update_parameters(info, frame.tick()) { + gain = self.gain.value(); + } + frame.mutate_with(|sample, _| *sample *= gain); + } + } + inputs + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::Gain => &mut self.gain, + _ => panic!("Unknown param {:?} for GainNode", id), + } + } +} diff --git a/components/media/audio/graph.rs b/components/media/audio/graph.rs new file mode 100644 index 00000000000..3e9e44c9768 --- /dev/null +++ b/components/media/audio/graph.rs @@ -0,0 +1,535 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cell::{RefCell, RefMut}; +use std::{cmp, fmt, hash}; + +use petgraph::Direction; +use petgraph::graph::DefaultIx; +use petgraph::stable_graph::{NodeIndex, StableGraph}; +use petgraph::visit::{DfsPostOrder, EdgeRef, Reversed}; +use smallvec::SmallVec; + +use crate::block::{Block, Chunk}; +use crate::destination_node::DestinationNode; +use crate::listener::AudioListenerNode; +use crate::node::{AudioNodeEngine, BlockInfo, ChannelCountMode, ChannelInterpretation}; +use crate::param::ParamType; + +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] +/// A unique identifier for nodes in the graph. Stable +/// under graph mutation. +pub struct NodeId(NodeIndex); + +impl NodeId { + pub fn input(self, port: u32) -> PortId { + PortId(self, PortIndex::Port(port)) + } + pub fn param(self, param: ParamType) -> PortId { + PortId(self, PortIndex::Param(param)) + } + pub fn output(self, port: u32) -> PortId { + PortId(self, PortIndex::Port(port)) + } + pub(crate) fn listener(self) -> PortId { + PortId(self, PortIndex::Listener(())) + } +} + +/// A zero-indexed "port" for a node. Most nodes have one +/// input and one output port, but some may have more. +/// For example, a channel splitter node will have one output +/// port for each channel. +/// +/// These are essentially indices into the Chunks +/// +/// Kind is a zero sized type and is useful for distinguishing +/// between input and output ports (which may otherwise share indices) +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] +pub enum PortIndex { + Port(u32), + Param(Kind::ParamId), + /// special variant only used for the implicit connection + /// from listeners to params + Listener(Kind::Listener), +} + +impl PortId { + pub fn node(&self) -> NodeId { + self.0 + } +} + +pub trait PortKind { + type ParamId: Copy + Eq + PartialEq + Ord + PartialOrd + hash::Hash + fmt::Debug; + type Listener: Copy + Eq + PartialEq + Ord + PartialOrd + hash::Hash + fmt::Debug; +} + +/// An identifier for a port. +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] +pub struct PortId(NodeId, PortIndex); + +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)] +/// Marker type for denoting that the port is an input port +/// of the node it is connected to +pub struct InputPort; +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)] +/// Marker type for denoting that the port is an output port +/// of the node it is connected to +pub struct OutputPort; + +impl PortKind for InputPort { + type ParamId = ParamType; + type Listener = (); +} + +#[derive(Debug, Hash, PartialOrd, Ord, PartialEq, Eq, Copy, Clone)] +pub enum Void {} + +impl PortKind for OutputPort { + // Params are only a feature of input ports. By using an empty type here + // we ensure that the PortIndex enum has zero overhead for outputs, + // taking up no extra discriminant space and eliminating PortIndex::Param + // branches entirely from the compiled code + type ParamId = Void; + type Listener = Void; +} + +pub struct AudioGraph { + graph: StableGraph, + dest_id: NodeId, + dests: Vec, + listener_id: NodeId, +} + +pub(crate) struct Node { + node: RefCell>, +} + +/// An edge in the graph +/// +/// This connects one or more pair of ports between two +/// nodes, each connection represented by a `Connection`. +/// WebAudio allows for multiple connections to/from the same port +/// however it does not allow for duplicate connections between pairs +/// of ports +pub(crate) struct Edge { + connections: SmallVec<[Connection; 1]>, +} + +impl Edge { + /// Find if there are connections between two given ports, return the index + fn has_between( + &self, + output_idx: PortIndex, + input_idx: PortIndex, + ) -> bool { + self.connections + .iter() + .any(|e| e.input_idx == input_idx && e.output_idx == output_idx) + } + + fn remove_by_output(&mut self, output_idx: PortIndex) { + self.connections.retain(|i| i.output_idx != output_idx) + } + + fn remove_by_input(&mut self, input_idx: PortIndex) { + self.connections.retain(|i| i.input_idx != input_idx) + } + + fn remove_by_pair( + &mut self, + output_idx: PortIndex, + input_idx: PortIndex, + ) { + self.connections + .retain(|i| i.output_idx != output_idx || i.input_idx != input_idx) + } +} + +/// A single connection between ports +struct Connection { + /// The index of the port on the input node + /// This is actually the /output/ of this edge + input_idx: PortIndex, + /// The index of the port on the output node + /// This is actually the /input/ of this edge + output_idx: PortIndex, + /// When the from node finishes processing, it will push + /// its data into this cache for the input node to read + cache: RefCell>, +} + +impl AudioGraph { + pub fn new(channel_count: u8) -> Self { + let mut graph = StableGraph::new(); + let dest_id = + NodeId(graph.add_node(Node::new(Box::new(DestinationNode::new(channel_count))))); + let listener_id = NodeId(graph.add_node(Node::new(Box::new(AudioListenerNode::new())))); + AudioGraph { + graph, + dest_id, + dests: vec![dest_id], + listener_id, + } + } + + /// Create a node, obtain its id + pub(crate) fn add_node(&mut self, node: Box) -> NodeId { + NodeId(self.graph.add_node(Node::new(node))) + } + + /// Connect an output port to an input port + /// + /// The edge goes *from* the output port *to* the input port, connecting two nodes + pub fn add_edge(&mut self, out: PortId, inp: PortId) { + let edge = self + .graph + .edges(out.node().0) + .find(|e| e.target() == inp.node().0) + .map(|e| e.id()); + if let Some(e) = edge { + // .find(|e| e.weight().has_between(out.1, inp.1)); + let w = self + .graph + .edge_weight_mut(e) + .expect("This edge is known to exist"); + if w.has_between(out.1, inp.1) { + return; + } + w.connections.push(Connection::new(inp.1, out.1)) + } else { + // add a new edge + self.graph + .add_edge(out.node().0, inp.node().0, Edge::new(inp.1, out.1)); + } + } + + /// Disconnect all outgoing connections from a node + /// + /// + pub fn disconnect_all_from(&mut self, node: NodeId) { + let edges = self.graph.edges(node.0).map(|e| e.id()).collect::>(); + for edge in edges { + self.graph.remove_edge(edge); + } + } + + /// Disconnect all outgoing connections from a node's output + /// + /// + pub fn disconnect_output(&mut self, out: PortId) { + let candidates: Vec<_> = self + .graph + .edges(out.node().0) + .map(|e| (e.id(), e.target())) + .collect(); + for (edge, to) in candidates { + let mut e = self + .graph + .remove_edge(edge) + .expect("Edge index is known to exist"); + e.remove_by_output(out.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, to, e); + } + } + } + + /// Disconnect connections from a node to another node + /// + /// + pub fn disconnect_between(&mut self, from: NodeId, to: NodeId) { + let edge = self + .graph + .edges(from.0) + .find(|e| e.target() == to.0) + .map(|e| e.id()); + if let Some(i) = edge { + self.graph.remove_edge(i); + } + } + + /// Disconnect all outgoing connections from a node's output to another node + /// + /// + pub fn disconnect_output_between(&mut self, out: PortId, to: NodeId) { + let edge = self + .graph + .edges(out.node().0) + .find(|e| e.target() == to.0) + .map(|e| e.id()); + if let Some(edge) = edge { + let mut e = self + .graph + .remove_edge(edge) + .expect("Edge index is known to exist"); + e.remove_by_output(out.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, to.0, e); + } + } + } + + /// Disconnect all outgoing connections from a node to another node's input + /// + /// Only used in WebAudio for disconnecting audio params + /// + /// + pub fn disconnect_to(&mut self, node: NodeId, inp: PortId) { + let edge = self + .graph + .edges(node.0) + .find(|e| e.target() == inp.node().0) + .map(|e| e.id()); + if let Some(edge) = edge { + let mut e = self + .graph + .remove_edge(edge) + .expect("Edge index is known to exist"); + e.remove_by_input(inp.1); + if !e.connections.is_empty() { + self.graph.add_edge(node.0, inp.node().0, e); + } + } + } + + /// Disconnect all outgoing connections from a node's output to another node's input + /// + /// + /// + pub fn disconnect_output_between_to( + &mut self, + out: PortId, + inp: PortId, + ) { + let edge = self + .graph + .edges(out.node().0) + .find(|e| e.target() == inp.node().0) + .map(|e| e.id()); + if let Some(edge) = edge { + let mut e = self + .graph + .remove_edge(edge) + .expect("Edge index is known to exist"); + e.remove_by_pair(out.1, inp.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, inp.node().0, e); + } + } + } + + /// Get the id of the destination node in this graph + /// + /// All graphs have a destination node, with one input port + pub fn dest_id(&self) -> NodeId { + self.dest_id + } + + /// Add additional terminator nodes + pub fn add_extra_dest(&mut self, dest: NodeId) { + self.dests.push(dest); + } + + /// Get the id of the AudioListener in this graph + /// + /// All graphs have a single listener, with no ports (but nine AudioParams) + /// + /// N.B. The listener actually has a single output port containing + /// its position data for the block, however this should + /// not be exposed to the DOM. + pub fn listener_id(&self) -> NodeId { + self.listener_id + } + + /// For a given block, process all the data on this graph + pub fn process(&mut self, info: &BlockInfo) -> Chunk { + // DFS post order: Children are processed before their parent, + // which is exactly what we need since the parent depends on the + // children's output + // + // This will only visit each node once + let reversed = Reversed(&self.graph); + + let mut blocks: SmallVec<[SmallVec<[Block; 1]>; 1]> = SmallVec::new(); + let mut output_counts: SmallVec<[u32; 1]> = SmallVec::new(); + + let mut visit = DfsPostOrder::empty(reversed); + + for dest in &self.dests { + visit.move_to(dest.0); + + while let Some(ix) = visit.next(reversed) { + let mut curr = self.graph[ix].node.borrow_mut(); + + let mut chunk = Chunk::default(); + chunk + .blocks + .resize(curr.input_count() as usize, Default::default()); + + // if we have inputs, collect all the computed blocks + // and construct a Chunk + + // set up scratch space to store all the blocks + blocks.clear(); + blocks.resize(curr.input_count() as usize, Default::default()); + + let mode = curr.channel_count_mode(); + let count = curr.channel_count(); + let interpretation = curr.channel_interpretation(); + + // all edges to this node are from its dependencies + for edge in self.graph.edges_directed(ix, Direction::Incoming) { + let edge = edge.weight(); + for connection in &edge.connections { + let mut block = connection + .cache + .borrow_mut() + .take() + .expect("Cache should have been filled from traversal"); + + match connection.input_idx { + PortIndex::Port(idx) => { + blocks[idx as usize].push(block); + }, + PortIndex::Param(param) => { + // param inputs are downmixed to mono + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + block.mix(1, ChannelInterpretation::Speakers); + curr.get_param(param).add_block(block) + }, + PortIndex::Listener(_) => curr.set_listenerdata(block), + } + } + } + + for (i, mut blocks) in blocks.drain(..).enumerate() { + if blocks.is_empty() { + if mode == ChannelCountMode::Explicit { + // It's silence, but mix it anyway + chunk.blocks[i].mix(count, interpretation); + } + } else if blocks.len() == 1 { + chunk.blocks[i] = blocks.pop().expect("`blocks` had length 1"); + match mode { + ChannelCountMode::Explicit => { + chunk.blocks[i].mix(count, interpretation); + }, + ChannelCountMode::ClampedMax => { + if chunk.blocks[i].chan_count() > count { + chunk.blocks[i].mix(count, interpretation); + } + }, + // It's one channel, it maxes itself + ChannelCountMode::Max => (), + } + } else { + let mix_count = match mode { + ChannelCountMode::Explicit => count, + _ => { + let mut max = 0; // max channel count + for block in &blocks { + max = cmp::max(max, block.chan_count()); + } + if mode == ChannelCountMode::ClampedMax { + max = cmp::min(max, count); + } + max + }, + }; + let block = blocks.into_iter().fold(Block::default(), |acc, mut block| { + block.mix(mix_count, interpretation); + acc.sum(block) + }); + chunk.blocks[i] = block; + } + } + + // actually run the node engine + let mut out = curr.process(chunk, info); + + assert_eq!(out.len(), curr.output_count() as usize); + if curr.output_count() == 0 { + continue; + } + + // Count how many output connections fan out from each port + // This is so that we don't have to needlessly clone audio buffers + // + // If this is inefficient, we can instead maintain this data + // cached on the node + output_counts.clear(); + output_counts.resize(curr.output_count() as usize, 0); + for edge in self.graph.edges(ix) { + let edge = edge.weight(); + for conn in &edge.connections { + if let PortIndex::Port(idx) = conn.output_idx { + output_counts[idx as usize] += 1; + } else { + unreachable!() + } + } + } + + // all the edges from this node go to nodes which depend on it, + // i.e. the nodes it outputs to. Store the blocks for retrieval. + for edge in self.graph.edges(ix) { + let edge = edge.weight(); + for conn in &edge.connections { + if let PortIndex::Port(idx) = conn.output_idx { + output_counts[idx as usize] -= 1; + // if there are no consumers left after this, take the data + let block = if output_counts[idx as usize] == 0 { + out[conn.output_idx].take() + } else { + out[conn.output_idx].clone() + }; + *conn.cache.borrow_mut() = Some(block); + } else { + unreachable!() + } + } + } + } + } + // The destination node stores its output on itself, extract it. + self.graph[self.dest_id.0] + .node + .borrow_mut() + .destination_data() + .expect("Destination node should have data cached") + } + + /// Obtain a mutable reference to a node + pub(crate) fn node_mut(&self, ix: NodeId) -> RefMut<'_, Box> { + self.graph[ix.0].node.borrow_mut() + } +} + +impl Node { + pub fn new(node: Box) -> Self { + Node { + node: RefCell::new(node), + } + } +} + +impl Edge { + pub fn new(input_idx: PortIndex, output_idx: PortIndex) -> Self { + Edge { + connections: SmallVec::from_buf([Connection::new(input_idx, output_idx)]), + } + } +} + +impl Connection { + pub fn new(input_idx: PortIndex, output_idx: PortIndex) -> Self { + Connection { + input_idx, + output_idx, + cache: RefCell::new(None), + } + } +} diff --git a/components/media/audio/iir_filter_node.rs b/components/media/audio/iir_filter_node.rs new file mode 100644 index 00000000000..7f92d0bc0a0 --- /dev/null +++ b/components/media/audio/iir_filter_node.rs @@ -0,0 +1,188 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::collections::VecDeque; +use std::sync::Arc; + +use log::warn; +use num_complex::Complex64; + +use crate::block::Chunk; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; + +const MAX_COEFFS: usize = 20; + +#[derive(Debug)] +pub struct IIRFilterNodeOptions { + pub feedforward: Arc>, + pub feedback: Arc>, +} + +#[derive(Clone)] +struct IIRFilter { + feedforward: Arc>, + feedback: Arc>, + inputs: VecDeque, + outputs: VecDeque, +} + +impl IIRFilter { + fn new(feedforward: Arc>, feedback: Arc>) -> Self { + Self { + feedforward, + feedback, + inputs: VecDeque::with_capacity(MAX_COEFFS), + outputs: VecDeque::with_capacity(MAX_COEFFS), + } + } + + fn calculate_output(&mut self, input: f32) -> f32 { + self.inputs.push_front(input as f64); + + if self.inputs.len() > MAX_COEFFS { + self.inputs.pop_back(); + } + + let inputs_sum = self + .feedforward + .iter() + .zip(self.inputs.iter()) + .fold(0.0, |acc, (c, v)| acc + c * v); + + let outputs_sum = self + .feedback + .iter() + .skip(1) + .zip(self.outputs.iter()) + .fold(0.0, |acc, (c, v)| acc + c * v); + + let output = (inputs_sum - outputs_sum) / self.feedback[0]; + + if output.is_nan() { + // Per spec: + // Note: The UA may produce a warning to notify the user that NaN values have occurred in the filter state. + // This is usually indicative of an unstable filter. + // + // But idk how to produce warnings + warn!("NaN in IIRFilter state"); + } + + self.outputs.push_front(output); + + if self.outputs.len() > MAX_COEFFS { + self.outputs.pop_back(); + } + + output as f32 + } +} + +#[derive(AudioNodeCommon)] +pub struct IIRFilterNode { + channel_info: ChannelInfo, + filters: Vec, +} + +impl IIRFilterNode { + pub fn new(options: IIRFilterNodeOptions, channel_info: ChannelInfo) -> Self { + debug_assert!( + !options.feedforward.is_empty(), + "NotSupportedError: feedforward must have at least one coeff" + ); + + debug_assert!( + options.feedforward.len() <= MAX_COEFFS, + "NotSupportedError: feedforward max length is {}", + MAX_COEFFS + ); + + debug_assert!( + options.feedforward.iter().any(|&v| v != 0.0_f64), + "InvalidStateError: all coeffs are zero" + ); + + debug_assert!( + !options.feedback.is_empty(), + "NotSupportedError: feedback must have at least one coeff" + ); + + debug_assert!( + options.feedback.len() <= MAX_COEFFS, + "NotSupportedError: feedback max length is {}", + MAX_COEFFS + ); + + debug_assert!( + options.feedback[0] != 0.0, + "InvalidStateError: first feedback coeff must not be zero" + ); + + let filter = IIRFilter::new(options.feedforward.clone(), options.feedback.clone()); + + Self { + filters: vec![filter; channel_info.computed_number_of_channels() as usize], + channel_info, + } + } + + pub fn get_frequency_response( + feedforward: &[f64], + feedback: &[f64], + frequency_hz: &[f32], + mag_response: &mut [f32], + phase_response: &mut [f32], + ) { + debug_assert!( + frequency_hz.len() == mag_response.len() && frequency_hz.len() == phase_response.len(), + "get_frequency_response params are of different length" + ); + + frequency_hz.iter().enumerate().for_each(|(idx, &f)| { + if !(0.0..1.0).contains(&f) { + mag_response[idx] = f32::NAN; + phase_response[idx] = f32::NAN; + } else { + let f = (-f as f64) * std::f64::consts::PI; + let z = Complex64::new(f64::cos(f), f64::sin(f)); + let numerator = Self::sum(feedforward, z); + let denominator = Self::sum(feedback, z); + + let response = numerator / denominator; + mag_response[idx] = response.norm() as f32; + phase_response[idx] = response.arg() as f32; + } + }); + } + + fn sum(coeffs: &[f64], z: Complex64) -> Complex64 { + coeffs.iter().fold(Complex64::new(0.0, 0.0), |acc, &coeff| { + acc * z + Complex64::new(coeff, 0.0) + }) + } +} + +impl AudioNodeEngine for IIRFilterNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::IIRFilterNode + } + + fn process(&mut self, inputs: Chunk, _info: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + let mut inputs = if inputs.blocks[0].is_silence() { + Chunk::explicit_silence() + } else { + inputs + }; + + let mut iter = inputs.blocks[0].iter(); + + while let Some(mut frame) = iter.next() { + frame.mutate_with(|sample, chan_idx| { + *sample = self.filters[chan_idx as usize].calculate_output(*sample); + }); + } + inputs + } +} diff --git a/components/media/audio/lib.rs b/components/media/audio/lib.rs new file mode 100644 index 00000000000..aa900fe6f71 --- /dev/null +++ b/components/media/audio/lib.rs @@ -0,0 +1,66 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate servo_media_derive; + +extern crate servo_media_player as player; + +extern crate byte_slice_cast; +extern crate euclid; +extern crate log; +extern crate num_complex; +extern crate num_traits; +extern crate petgraph; +extern crate smallvec; +extern crate speexdsp_resampler; +#[macro_use] +pub mod macros; +extern crate servo_media_streams; +extern crate servo_media_traits; + +pub mod analyser_node; +pub mod biquad_filter_node; +pub mod block; +pub mod buffer_source_node; +pub mod channel_node; +pub mod constant_source_node; +pub mod context; +pub mod decoder; +pub mod destination_node; +pub mod gain_node; +pub mod graph; +pub mod iir_filter_node; +pub mod listener; +pub mod media_element_source_node; +pub mod media_stream_destination_node; +pub mod media_stream_source_node; +pub mod node; +pub mod offline_sink; +pub mod oscillator_node; +pub mod panner_node; +pub mod param; +pub mod render_thread; +pub mod sink; +pub mod stereo_panner; +pub mod wave_shaper_node; + +pub trait AudioBackend { + type Sink: sink::AudioSink + 'static; + fn make_decoder() -> Box; + fn make_sink() -> Result; + fn make_streamreader( + id: servo_media_streams::MediaStreamId, + sample_rate: f32, + ) -> Box; +} + +pub trait AudioStreamReader { + fn pull(&self) -> block::Block; + fn start(&self); + fn stop(&self); +} diff --git a/components/media/audio/listener.rs b/components/media/audio/listener.rs new file mode 100644 index 00000000000..10ea5249097 --- /dev/null +++ b/components/media/audio/listener.rs @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::block::{Block, Chunk}; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; +use crate::param::{Param, ParamDir, ParamType}; + +/// AudioListeners are fake nodes; from the user's point of view they're +/// a non-node entity with zero inputs and outputs, but with AudioParams +/// that can be manipulated. +/// +/// Internally, PannerNodes all have an implicit PortIndex::Listener connection +/// from a hidden output port on AudioListeners that contains all the position data. +/// +/// This encodes the otherwise implicit dependency between AudioListeners and PannerNodes +/// so that if there is a cycle involving panner nodes and the audio params on the listener, +/// the cycle breaking algorithm can deal with it. +#[derive(AudioNodeCommon)] +pub(crate) struct AudioListenerNode { + channel_info: ChannelInfo, + position_x: Param, + position_y: Param, + position_z: Param, + forward_x: Param, + forward_y: Param, + forward_z: Param, + up_x: Param, + up_y: Param, + up_z: Param, +} + +impl AudioListenerNode { + pub fn new() -> Self { + Self { + channel_info: Default::default(), + position_x: Param::new(0.), + position_y: Param::new(0.), + position_z: Param::new(0.), + forward_x: Param::new(0.), + forward_y: Param::new(0.), + forward_z: Param::new(-1.), + up_x: Param::new(0.), + up_y: Param::new(1.), + up_z: Param::new(0.), + } + } +} + +impl AudioNodeEngine for AudioListenerNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::AudioListenerNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.is_empty()); + + // XXXManishearth in the common case when all of these are constant, + // it would be nice to instead send just the constant values down + let mut block = Block::for_channels_explicit(9); + self.position_x.flush_to_block(info, block.data_chan_mut(0)); + self.position_y.flush_to_block(info, block.data_chan_mut(1)); + self.position_z.flush_to_block(info, block.data_chan_mut(2)); + self.forward_x.flush_to_block(info, block.data_chan_mut(3)); + self.forward_y.flush_to_block(info, block.data_chan_mut(4)); + self.forward_z.flush_to_block(info, block.data_chan_mut(5)); + self.up_x.flush_to_block(info, block.data_chan_mut(6)); + self.up_y.flush_to_block(info, block.data_chan_mut(7)); + self.up_z.flush_to_block(info, block.data_chan_mut(8)); + + inputs.blocks.push(block); + inputs + } + + fn input_count(&self) -> u32 { + 0 + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::Position(ParamDir::X) => &mut self.position_x, + ParamType::Position(ParamDir::Y) => &mut self.position_y, + ParamType::Position(ParamDir::Z) => &mut self.position_z, + ParamType::Forward(ParamDir::X) => &mut self.forward_x, + ParamType::Forward(ParamDir::Y) => &mut self.forward_y, + ParamType::Forward(ParamDir::Z) => &mut self.forward_z, + ParamType::Up(ParamDir::X) => &mut self.up_x, + ParamType::Up(ParamDir::Y) => &mut self.up_y, + ParamType::Up(ParamDir::Z) => &mut self.up_z, + _ => panic!("Unknown param {:?} for AudioListenerNode", id), + } + } +} diff --git a/components/media/audio/macros.rs b/components/media/audio/macros.rs new file mode 100644 index 00000000000..191df66e8c4 --- /dev/null +++ b/components/media/audio/macros.rs @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#[macro_export] +macro_rules! make_message_handler( + ( + $( + $node:ident: $handler:ident + ),+ + ) => ( + fn message_specific(&mut self, msg: $crate::node::AudioNodeMessage, sample_rate: f32) { + match msg { + $($crate::node::AudioNodeMessage::$node(m) => self.$handler(m, sample_rate)),+, + _ => (), + } + } + ); +); + +#[macro_export] +macro_rules! make_state_change( + ($fn_name:ident, $state:ident, $render_msg:ident) => ( + pub fn $fn_name(&self) -> StateChangeResult { + self.state.set(ProcessingState::$state); + let (tx, rx) = mpsc::channel(); + let _ = self.sender.send(AudioRenderThreadMsg::$render_msg(tx)); + rx.recv().unwrap() + } + ); +); + +#[macro_export] +macro_rules! make_render_thread_state_change( + ($fn_name:ident, $state:ident, $sink_method:ident) => ( + fn $fn_name(&mut self) -> StateChangeResult { + if self.state == ProcessingState::$state { + return Some(()); + } + self.state = ProcessingState::$state; + self.sink.$sink_method().ok() + } + ); +); diff --git a/components/media/audio/media_element_source_node.rs b/components/media/audio/media_element_source_node.rs new file mode 100644 index 00000000000..7aa70b3d154 --- /dev/null +++ b/components/media/audio/media_element_source_node.rs @@ -0,0 +1,136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::mpsc::Sender; +use std::sync::{Arc, Mutex}; + +use player::audio::AudioRenderer; + +use crate::block::{Block, Chunk, FRAMES_PER_BLOCK}; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; + +#[derive(Debug, Clone)] +pub enum MediaElementSourceNodeMessage { + GetAudioRenderer(Sender>>), +} + +#[derive(AudioNodeCommon)] +pub(crate) struct MediaElementSourceNode { + channel_info: ChannelInfo, + renderer: Arc>, + buffers: Arc>>>, + playback_offset: usize, +} + +impl MediaElementSourceNode { + pub fn new(channel_info: ChannelInfo) -> Self { + let buffers = Arc::new(Mutex::new(Vec::new())); + let renderer = Arc::new(Mutex::new(MediaElementSourceNodeRenderer::new( + buffers.clone(), + ))); + Self { + channel_info, + renderer, + buffers, + playback_offset: 0, + } + } + + pub fn handle_message(&mut self, message: MediaElementSourceNodeMessage, _: f32) { + match message { + MediaElementSourceNodeMessage::GetAudioRenderer(sender) => { + let _ = sender.send(self.renderer.clone()); + }, + } + } +} + +impl AudioNodeEngine for MediaElementSourceNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::MediaElementSourceNode + } + + fn process(&mut self, mut inputs: Chunk, _info: &BlockInfo) -> Chunk { + debug_assert!(inputs.is_empty()); + + let buffers = self.buffers.lock().unwrap(); + let chans = buffers.len() as u8; + + if chans == 0 { + inputs.blocks.push(Default::default()); + return inputs; + } + + let len = buffers[0].len(); + + let frames_per_block = FRAMES_PER_BLOCK.0 as usize; + let samples_to_copy = if self.playback_offset + frames_per_block > len { + len - self.playback_offset + } else { + frames_per_block + }; + let next_offset = self.playback_offset + samples_to_copy; + if samples_to_copy == FRAMES_PER_BLOCK.0 as usize { + // copy entire chan + let mut block = Block::empty(); + for chan in 0..chans { + block.push_chan(&buffers[chan as usize][self.playback_offset..next_offset]); + } + inputs.blocks.push(block) + } else { + // silent fill and copy + let mut block = Block::default(); + block.repeat(chans); + block.explicit_repeat(); + for chan in 0..chans { + let data = block.data_chan_mut(chan); + let (_, data) = data.split_at_mut(0); + let (data, _) = data.split_at_mut(samples_to_copy); + data.copy_from_slice(&buffers[chan as usize][self.playback_offset..next_offset]); + } + inputs.blocks.push(block) + } + + self.playback_offset = next_offset; + + inputs + } + + fn input_count(&self) -> u32 { + 0 + } + + make_message_handler!(MediaElementSourceNode: handle_message); +} + +struct MediaElementSourceNodeRenderer { + buffers: Arc>>>, + channels: HashMap, +} + +impl MediaElementSourceNodeRenderer { + pub fn new(buffers: Arc>>>) -> Self { + Self { + buffers, + channels: HashMap::new(), + } + } +} + +impl AudioRenderer for MediaElementSourceNodeRenderer { + fn render(&mut self, sample: Box>, channel_pos: u32) { + let channel = match self.channels.entry(channel_pos) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let mut buffers = self.buffers.lock().unwrap(); + let len = buffers.len(); + buffers.resize(len + 1, Vec::new()); + *entry.insert(buffers.len()) + }, + }; + self.buffers.lock().unwrap()[channel - 1].extend_from_slice((*sample).as_ref()); + } +} diff --git a/components/media/audio/media_stream_destination_node.rs b/components/media/audio/media_stream_destination_node.rs new file mode 100644 index 00000000000..1cef42e892e --- /dev/null +++ b/components/media/audio/media_stream_destination_node.rs @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use servo_media_streams::MediaSocket; + +use crate::block::Chunk; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; +use crate::sink::AudioSink; + +#[derive(AudioNodeCommon)] +pub(crate) struct MediaStreamDestinationNode { + channel_info: ChannelInfo, + sink: Box, +} + +impl MediaStreamDestinationNode { + pub fn new( + socket: Box, + sample_rate: f32, + sink: Box, + channel_info: ChannelInfo, + ) -> Self { + sink.init_stream(channel_info.count, sample_rate, socket) + .expect("init_stream failed"); + sink.play().expect("Sink didn't start"); + MediaStreamDestinationNode { channel_info, sink } + } +} + +impl AudioNodeEngine for MediaStreamDestinationNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::MediaStreamDestinationNode + } + + fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk { + self.sink + .push_data(inputs) + .expect("Pushing to stream failed"); + Chunk::default() + } + + fn output_count(&self) -> u32 { + 0 + } +} diff --git a/components/media/audio/media_stream_source_node.rs b/components/media/audio/media_stream_source_node.rs new file mode 100644 index 00000000000..7cd3164affa --- /dev/null +++ b/components/media/audio/media_stream_source_node.rs @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::AudioStreamReader; +use crate::block::Chunk; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; +use crate::param::{Param, ParamType}; + +#[derive(AudioNodeCommon)] +pub(crate) struct MediaStreamSourceNode { + channel_info: ChannelInfo, + reader: Box, + playing: bool, +} + +impl MediaStreamSourceNode { + pub fn new(reader: Box, channel_info: ChannelInfo) -> Self { + Self { + channel_info, + reader, + playing: false, + } + } +} + +impl AudioNodeEngine for MediaStreamSourceNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::MediaStreamSourceNode + } + + fn process(&mut self, mut inputs: Chunk, _: &BlockInfo) -> Chunk { + debug_assert!(inputs.is_empty()); + + if !self.playing { + self.playing = true; + self.reader.start(); + } + + let block = self.reader.pull(); + inputs.blocks.push(block); + + inputs + } + + fn input_count(&self) -> u32 { + 0 + } + + fn get_param(&mut self, _: ParamType) -> &mut Param { + panic!("No params on MediaStreamSourceNode"); + } +} diff --git a/components/media/audio/node.rs b/components/media/audio/node.rs new file mode 100644 index 00000000000..a59d8157275 --- /dev/null +++ b/components/media/audio/node.rs @@ -0,0 +1,250 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cmp::min; +use std::sync::mpsc::Sender; + +use servo_media_streams::{MediaSocket, MediaStreamId}; + +use crate::biquad_filter_node::{BiquadFilterNodeMessage, BiquadFilterNodeOptions}; +use crate::block::{Block, Chunk, Tick}; +use crate::buffer_source_node::{AudioBufferSourceNodeMessage, AudioBufferSourceNodeOptions}; +use crate::channel_node::ChannelNodeOptions; +use crate::constant_source_node::ConstantSourceNodeOptions; +use crate::gain_node::GainNodeOptions; +use crate::iir_filter_node::IIRFilterNodeOptions; +use crate::media_element_source_node::MediaElementSourceNodeMessage; +use crate::oscillator_node::{OscillatorNodeMessage, OscillatorNodeOptions}; +use crate::panner_node::{PannerNodeMessage, PannerNodeOptions}; +use crate::param::{Param, ParamRate, ParamType, UserAutomationEvent}; +use crate::stereo_panner::StereoPannerOptions; +use crate::wave_shaper_node::{WaveShaperNodeMessage, WaveShaperNodeOptions}; + +/// Information required to construct an audio node +pub enum AudioNodeInit { + AnalyserNode(Box), + BiquadFilterNode(BiquadFilterNodeOptions), + AudioBuffer, + AudioBufferSourceNode(AudioBufferSourceNodeOptions), + ChannelMergerNode(ChannelNodeOptions), + ChannelSplitterNode, + ConstantSourceNode(ConstantSourceNodeOptions), + ConvolverNode, + DelayNode, + DynamicsCompressionNode, + GainNode(GainNodeOptions), + IIRFilterNode(IIRFilterNodeOptions), + MediaElementSourceNode, + MediaStreamDestinationNode(Box), + MediaStreamSourceNode(MediaStreamId), + OscillatorNode(OscillatorNodeOptions), + PannerNode(PannerNodeOptions), + PeriodicWave, + ScriptProcessorNode, + StereoPannerNode(StereoPannerOptions), + WaveShaperNode(WaveShaperNodeOptions), +} + +/// Type of AudioNodeEngine. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioNodeType { + /// Not a constructable node + AudioListenerNode, + AnalyserNode, + BiquadFilterNode, + AudioBuffer, + AudioBufferSourceNode, + ChannelMergerNode, + ChannelSplitterNode, + ConstantSourceNode, + ConvolverNode, + DelayNode, + DestinationNode, + DynamicsCompressionNode, + GainNode, + IIRFilterNode, + MediaElementSourceNode, + MediaStreamDestinationNode, + MediaStreamSourceNode, + OscillatorNode, + PannerNode, + PeriodicWave, + ScriptProcessorNode, + StereoPannerNode, + WaveShaperNode, +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum ChannelCountMode { + Max, + ClampedMax, + Explicit, +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum ChannelInterpretation { + Discrete, + Speakers, +} + +#[derive(Copy, Clone)] +pub struct BlockInfo { + pub sample_rate: f32, + pub frame: Tick, + pub time: f64, +} + +impl BlockInfo { + /// Given the current block, calculate the absolute zero-relative + /// tick of the given tick + pub fn absolute_tick(&self, tick: Tick) -> Tick { + self.frame + tick + } +} + +pub struct ChannelInfo { + pub count: u8, + pub mode: ChannelCountMode, + pub interpretation: ChannelInterpretation, + pub context_channel_count: u8, +} + +impl Default for ChannelInfo { + fn default() -> Self { + ChannelInfo { + count: 2, + mode: ChannelCountMode::Max, + interpretation: ChannelInterpretation::Speakers, + context_channel_count: 2, + } + } +} + +impl ChannelInfo { + /// + pub fn computed_number_of_channels(&self) -> u8 { + match self.mode { + ChannelCountMode::Max => self.context_channel_count, + ChannelCountMode::ClampedMax => min(self.count, self.context_channel_count), + ChannelCountMode::Explicit => self.count, + } + } +} + +pub(crate) trait AudioNodeCommon { + fn channel_info(&self) -> &ChannelInfo; + + fn channel_info_mut(&mut self) -> &mut ChannelInfo; +} + +/// This trait represents the common features of all audio nodes. +pub(crate) trait AudioNodeEngine: Send + AudioNodeCommon { + fn node_type(&self) -> AudioNodeType; + + fn process(&mut self, inputs: Chunk, info: &BlockInfo) -> Chunk; + + fn message(&mut self, msg: AudioNodeMessage, sample_rate: f32) { + match msg { + AudioNodeMessage::GetParamValue(id, tx) => { + let _ = tx.send(self.get_param(id).value()); + }, + AudioNodeMessage::SetChannelCount(c) => self.set_channel_count(c), + AudioNodeMessage::SetChannelMode(c) => self.set_channel_count_mode(c), + AudioNodeMessage::SetChannelInterpretation(c) => self.set_channel_interpretation(c), + AudioNodeMessage::SetParam(id, event) => self + .get_param(id) + .insert_event(event.convert_to_event(sample_rate)), + AudioNodeMessage::SetParamRate(id, rate) => self.get_param(id).set_rate(rate), + _ => self.message_specific(msg, sample_rate), + } + } + + /// Messages specific to this node + fn message_specific(&mut self, _: AudioNodeMessage, _sample_rate: f32) {} + + fn input_count(&self) -> u32 { + 1 + } + fn output_count(&self) -> u32 { + 1 + } + + /// Number of input channels for each input port + fn channel_count(&self) -> u8 { + self.channel_info().count + } + + fn channel_count_mode(&self) -> ChannelCountMode { + self.channel_info().mode + } + + fn channel_interpretation(&self) -> ChannelInterpretation { + self.channel_info().interpretation + } + + fn set_channel_interpretation(&mut self, i: ChannelInterpretation) { + self.channel_info_mut().interpretation = i + } + fn set_channel_count(&mut self, c: u8) { + self.channel_info_mut().count = c; + } + fn set_channel_count_mode(&mut self, m: ChannelCountMode) { + self.channel_info_mut().mode = m; + } + + /// If we're the destination node, extract the contained data + fn destination_data(&mut self) -> Option { + None + } + + fn get_param(&mut self, _: ParamType) -> &mut Param { + panic!("No params on node {:?}", self.node_type()) + } + + fn set_listenerdata(&mut self, _: Block) { + panic!("can't accept listener connections") + } +} + +pub enum AudioNodeMessage { + AudioBufferSourceNode(AudioBufferSourceNodeMessage), + AudioScheduledSourceNode(AudioScheduledSourceNodeMessage), + BiquadFilterNode(BiquadFilterNodeMessage), + GetParamValue(ParamType, Sender), + MediaElementSourceNode(MediaElementSourceNodeMessage), + OscillatorNode(OscillatorNodeMessage), + PannerNode(PannerNodeMessage), + SetChannelCount(u8), + SetChannelMode(ChannelCountMode), + SetChannelInterpretation(ChannelInterpretation), + SetParam(ParamType, UserAutomationEvent), + SetParamRate(ParamType, ParamRate), + WaveShaperNode(WaveShaperNodeMessage), +} + +pub struct OnEndedCallback(pub Box); + +impl OnEndedCallback { + pub fn new(callback: F) -> Self { + OnEndedCallback(Box::new(callback)) + } +} + +/// Type of message directed to AudioScheduledSourceNodes. +pub enum AudioScheduledSourceNodeMessage { + /// Schedules a sound to playback at an exact time. + Start(f64), + /// Schedules a sound to stop playback at an exact time. + Stop(f64), + /// Register onended event callback. + RegisterOnEndedCallback(OnEndedCallback), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ShouldPlay { + /// Don't play anything + No, + /// Play, given start and end tick offsets + Between(Tick, Tick), +} diff --git a/components/media/audio/offline_sink.rs b/components/media/audio/offline_sink.rs new file mode 100644 index 00000000000..bb7a848d186 --- /dev/null +++ b/components/media/audio/offline_sink.rs @@ -0,0 +1,109 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cell::{Cell, RefCell}; +use std::sync::mpsc::Sender; + +use servo_media_streams::MediaSocket; + +use crate::block::{Chunk, FRAMES_PER_BLOCK_USIZE}; +use crate::render_thread::{AudioRenderThreadMsg, SinkEosCallback}; +use crate::sink::{AudioSink, AudioSinkError}; + +pub struct ProcessedAudio(Box<[f32]>); + +impl AsRef<[f32]> for ProcessedAudio { + fn as_ref(&self) -> &[f32] { + &self.0 + } +} + +pub struct OfflineAudioSink { + buffer: RefCell>>, + channel_count: usize, + has_enough_data: Cell, + length: usize, + rendered_blocks: Cell, + eos_callback: RefCell>, +} + +impl OfflineAudioSink { + pub fn new(channel_count: usize, length: usize) -> Self { + Self { + buffer: RefCell::new(None), + channel_count, + has_enough_data: Cell::new(false), + length, + rendered_blocks: Cell::new(0), + eos_callback: RefCell::new(None), + } + } +} + +impl AudioSink for OfflineAudioSink { + fn init(&self, _: f32, _: Sender) -> Result<(), AudioSinkError> { + Ok(()) + } + fn init_stream(&self, _: u8, _: f32, _: Box) -> Result<(), AudioSinkError> { + unreachable!("OfflineAudioSink should never be used for MediaStreamDestinationNode") + } + fn play(&self) -> Result<(), AudioSinkError> { + self.has_enough_data.set(false); + Ok(()) + } + + fn stop(&self) -> Result<(), AudioSinkError> { + self.has_enough_data.set(true); + Ok(()) + } + + fn has_enough_data(&self) -> bool { + self.has_enough_data.get() || + (self.rendered_blocks.get() * FRAMES_PER_BLOCK_USIZE >= self.length) + } + + fn push_data(&self, mut chunk: Chunk) -> Result<(), AudioSinkError> { + let offset = self.rendered_blocks.get() * FRAMES_PER_BLOCK_USIZE; + let (last, copy_len) = if self.length - offset <= FRAMES_PER_BLOCK_USIZE { + (true, self.length - offset) + } else { + (false, FRAMES_PER_BLOCK_USIZE) + }; + let mut buffer = self.buffer.borrow_mut(); + if buffer.is_none() { + *buffer = Some(vec![0.; self.channel_count * self.length]); + } + if chunk.is_empty() { + chunk.blocks.push(Default::default()); + } + if chunk.blocks[0].is_empty() { + chunk.blocks[0].explicit_silence(); + } + if let Some(ref mut buffer) = *buffer { + for channel_number in 0..self.channel_count { + let channel_offset = offset + (channel_number * self.length); + let channel_data = &mut buffer[channel_offset..channel_offset + copy_len]; + channel_data + .copy_from_slice(&chunk.blocks[0].data_chan(channel_number as u8)[0..copy_len]); + } + }; + self.rendered_blocks.set(self.rendered_blocks.get() + 1); + + if last { + if let Some(callback) = self.eos_callback.borrow_mut().take() { + let processed_audio = ProcessedAudio(buffer.take().unwrap().into_boxed_slice()); + callback(Box::new(processed_audio)); + } + } + + Ok(()) + } + + fn set_eos_callback( + &self, + callback: Box>) + Send + Sync + 'static>, + ) { + *self.eos_callback.borrow_mut() = Some(callback); + } +} diff --git a/components/media/audio/oscillator_node.rs b/components/media/audio/oscillator_node.rs new file mode 100644 index 00000000000..7d9fd2da123 --- /dev/null +++ b/components/media/audio/oscillator_node.rs @@ -0,0 +1,196 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use num_traits::cast::NumCast; + +use crate::block::{Chunk, Tick}; +use crate::node::{ + AudioNodeEngine, AudioNodeType, AudioScheduledSourceNodeMessage, BlockInfo, ChannelInfo, + OnEndedCallback, ShouldPlay, +}; +use crate::param::{Param, ParamType}; + +#[derive(Clone, Debug)] +pub struct PeriodicWaveOptions { + // XXX https://webaudio.github.io/web-audio-api/#dictdef-periodicwaveoptions +} +#[derive(Clone, Debug)] +pub enum OscillatorType { + Sine, + Square, + Sawtooth, + Triangle, + Custom, +} + +#[derive(Clone, Debug)] +pub struct OscillatorNodeOptions { + pub oscillator_type: OscillatorType, + pub freq: f32, + pub detune: f32, + pub periodic_wave_options: Option, +} + +impl Default for OscillatorNodeOptions { + fn default() -> Self { + OscillatorNodeOptions { + oscillator_type: OscillatorType::Sine, + freq: 440., + detune: 0., + periodic_wave_options: None, + } + } +} + +#[derive(Clone, Debug)] +pub enum OscillatorNodeMessage { + SetOscillatorType(OscillatorType), +} + +#[derive(AudioScheduledSourceNode, AudioNodeCommon)] +pub(crate) struct OscillatorNode { + channel_info: ChannelInfo, + oscillator_type: OscillatorType, + frequency: Param, + detune: Param, + phase: f64, + /// Time at which the source should start playing. + start_at: Option, + /// Time at which the source should stop playing. + stop_at: Option, + /// The ended event callback. + onended_callback: Option, +} + +impl OscillatorNode { + pub fn new(options: OscillatorNodeOptions, channel_info: ChannelInfo) -> Self { + Self { + channel_info, + oscillator_type: options.oscillator_type, + frequency: Param::new(options.freq), + detune: Param::new(options.detune), + phase: 0., + start_at: None, + stop_at: None, + onended_callback: None, + } + } + + pub fn update_parameters(&mut self, info: &BlockInfo, tick: Tick) -> bool { + self.frequency.update(info, tick) + } + + fn handle_oscillator_message(&mut self, message: OscillatorNodeMessage, _sample_rate: f32) { + match message { + OscillatorNodeMessage::SetOscillatorType(o) => { + self.oscillator_type = o; + }, + } + } +} + +impl AudioNodeEngine for OscillatorNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::OscillatorNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + // XXX Implement this properly and according to self.options + // as defined in https://webaudio.github.io/web-audio-api/#oscillatornode + use std::f64::consts::PI; + debug_assert!(inputs.is_empty()); + inputs.blocks.push(Default::default()); + let (start_at, stop_at) = match self.should_play_at(info.frame) { + ShouldPlay::No => { + return inputs; + }, + ShouldPlay::Between(start, end) => (start, end), + }; + + { + inputs.blocks[0].explicit_silence(); + let mut iter = inputs.blocks[0].iter(); + + // Convert all our parameters to the target type for calculations + let vol: f32 = 1.0; + let sample_rate = info.sample_rate as f64; + let two_pi = 2.0 * PI; + + // We're carrying a phase with up to 2pi around instead of working + // on the sample offset. High sample offsets cause too much inaccuracy when + // converted to floating point numbers and then iterated over in 1-steps + // + // Also, if the frequency changes the phase should not + let mut step = two_pi * self.frequency.value() as f64 / sample_rate; + while let Some(mut frame) = iter.next() { + let tick = frame.tick(); + if tick < start_at { + continue; + } else if tick > stop_at { + break; + } + + if self.update_parameters(info, tick) { + step = two_pi * self.frequency.value() as f64 / sample_rate; + } + let mut value = vol; + match self.oscillator_type { + OscillatorType::Sine => { + value = vol * f32::sin(NumCast::from(self.phase).unwrap()); + }, + + OscillatorType::Square => { + if self.phase >= PI && self.phase < two_pi { + value = vol * 1.0; + } else if self.phase > 0.0 && self.phase < PI { + value = -vol; + } + }, + + OscillatorType::Sawtooth => { + value = vol * (self.phase / (PI)) as f32; + }, + + OscillatorType::Triangle => { + if self.phase >= 0. && self.phase < PI / 2. { + value = vol * 2.0 * (self.phase / (PI)) as f32; + } else if self.phase >= PI / 2. && self.phase < PI { + value = vol * (1. - ((self.phase - (PI / 2.)) * (2. / PI)) as f32); + } else if self.phase >= PI && self.phase < (3. * PI / 2.) { + value = -vol * (1. - ((self.phase - (PI / 2.)) * (2. / PI)) as f32); + } else if self.phase >= 3. * PI / 2. && self.phase < 2. * PI { + value = vol * (-2.0) * (self.phase / (PI)) as f32; + } + }, + + OscillatorType::Custom => {}, + } + + frame.mutate_with(|sample, _| *sample = value); + + self.phase += step; + if self.phase >= two_pi { + self.phase -= two_pi; + } + } + } + inputs + } + + fn input_count(&self) -> u32 { + 0 + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::Frequency => &mut self.frequency, + ParamType::Detune => &mut self.detune, + _ => panic!("Unknown param {:?} for OscillatorNode", id), + } + } + make_message_handler!( + AudioScheduledSourceNode: handle_source_node_message, + OscillatorNode: handle_oscillator_message + ); +} diff --git a/components/media/audio/panner_node.rs b/components/media/audio/panner_node.rs new file mode 100644 index 00000000000..fd1e2df7cd5 --- /dev/null +++ b/components/media/audio/panner_node.rs @@ -0,0 +1,410 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::f32::consts::PI; + +use euclid::default::Vector3D; + +use crate::block::{Block, Chunk, FRAMES_PER_BLOCK, Tick}; +use crate::node::{AudioNodeEngine, AudioNodeMessage, AudioNodeType, BlockInfo, ChannelInfo}; +use crate::param::{Param, ParamDir, ParamType}; + +// .normalize(), but it takes into account zero vectors +pub fn normalize_zero(v: Vector3D) -> Vector3D { + let len = v.length(); + if len == 0. { v } else { v / len } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum PanningModel { + EqualPower, + HRTF, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum DistanceModel { + Linear, + Inverse, + Exponential, +} + +#[derive(Copy, Clone, Debug)] +pub struct PannerNodeOptions { + pub panning_model: PanningModel, + pub distance_model: DistanceModel, + pub position_x: f32, + pub position_y: f32, + pub position_z: f32, + pub orientation_x: f32, + pub orientation_y: f32, + pub orientation_z: f32, + pub ref_distance: f64, + pub max_distance: f64, + pub rolloff_factor: f64, + pub cone_inner_angle: f64, + pub cone_outer_angle: f64, + pub cone_outer_gain: f64, +} + +pub enum PannerNodeMessage { + SetPanningModel(PanningModel), + SetDistanceModel(DistanceModel), + SetRefDistance(f64), + SetMaxDistance(f64), + SetRolloff(f64), + SetConeInner(f64), + SetConeOuter(f64), + SetConeGain(f64), +} + +impl Default for PannerNodeOptions { + fn default() -> Self { + PannerNodeOptions { + panning_model: PanningModel::EqualPower, + distance_model: DistanceModel::Inverse, + position_x: 0., + position_y: 0., + position_z: 0., + orientation_x: 1., + orientation_y: 0., + orientation_z: 0., + ref_distance: 1., + max_distance: 10000., + rolloff_factor: 1., + cone_inner_angle: 360., + cone_outer_angle: 360., + cone_outer_gain: 0., + } + } +} + +#[derive(AudioNodeCommon)] +pub(crate) struct PannerNode { + channel_info: ChannelInfo, + panning_model: PanningModel, + distance_model: DistanceModel, + position_x: Param, + position_y: Param, + position_z: Param, + orientation_x: Param, + orientation_y: Param, + orientation_z: Param, + ref_distance: f64, + max_distance: f64, + rolloff_factor: f64, + cone_inner_angle: f64, + cone_outer_angle: f64, + cone_outer_gain: f64, + listener_data: Option, +} + +impl PannerNode { + pub fn new(options: PannerNodeOptions, channel_info: ChannelInfo) -> Self { + if options.panning_model == PanningModel::HRTF { + log::warn!("HRTF requested but not supported") + } + Self { + channel_info, + panning_model: options.panning_model, + distance_model: options.distance_model, + position_x: Param::new(options.position_x), + position_y: Param::new(options.position_y), + position_z: Param::new(options.position_z), + orientation_x: Param::new(options.orientation_x), + orientation_y: Param::new(options.orientation_y), + orientation_z: Param::new(options.orientation_z), + ref_distance: options.ref_distance, + max_distance: options.max_distance, + rolloff_factor: options.rolloff_factor, + cone_inner_angle: options.cone_inner_angle, + cone_outer_angle: options.cone_outer_angle, + cone_outer_gain: options.cone_outer_gain, + listener_data: None, + } + } + + pub fn update_parameters(&mut self, info: &BlockInfo, tick: Tick) -> bool { + let mut changed = self.position_x.update(info, tick); + changed |= self.position_y.update(info, tick); + changed |= self.position_z.update(info, tick); + changed |= self.orientation_x.update(info, tick); + changed |= self.orientation_y.update(info, tick); + changed |= self.orientation_z.update(info, tick); + changed + } + + /// Computes azimuth, elevation, and distance of source with respect to a + /// given AudioListener's position, forward, and up vectors + /// in degrees + /// + /// + /// + fn azimuth_elevation_distance( + &self, + listener: (Vector3D, Vector3D, Vector3D), + ) -> (f32, f32, f64) { + let (listener_position, listener_forward, listener_up) = listener; + let source_position = Vector3D::new( + self.position_x.value(), + self.position_y.value(), + self.position_z.value(), + ); + + // degenerate case + if source_position == listener_position { + return (0., 0., 0.); + } + + let diff = source_position - listener_position; + let distance = diff.length(); + let source_listener = normalize_zero(diff); + let listener_right = listener_forward.cross(listener_up); + let listener_right_norm = normalize_zero(listener_right); + let listener_forward_norm = normalize_zero(listener_forward); + + let up = listener_right_norm.cross(listener_forward_norm); + + let up_projection = source_listener.dot(up); + let projected_source = normalize_zero(source_listener - up * up_projection); + let mut azimuth = 180. * projected_source.dot(listener_right_norm).acos() / PI; + + let front_back = projected_source.dot(listener_forward_norm); + if front_back < 0. { + azimuth = 360. - azimuth; + } + if (0. ..=270.).contains(&azimuth) { + azimuth = 90. - azimuth; + } else { + azimuth = 450. - azimuth; + } + + let mut elevation = 90. - 180. * source_listener.dot(up).acos() / PI; + + if elevation > 90. { + elevation = 180. - elevation; + } else if elevation < -90. { + elevation = -180. - elevation; + } + + (azimuth, elevation, distance as f64) + } + + /// + fn cone_gain(&self, listener: (Vector3D, Vector3D, Vector3D)) -> f64 { + let (listener_position, _, _) = listener; + let source_position = Vector3D::new( + self.position_x.value(), + self.position_y.value(), + self.position_z.value(), + ); + let source_orientation = Vector3D::new( + self.orientation_x.value(), + self.orientation_y.value(), + self.orientation_z.value(), + ); + + if source_orientation == Vector3D::zero() || + (self.cone_inner_angle == 360. && self.cone_outer_angle == 360.) + { + return 0.; + } + + let normalized_source_orientation = normalize_zero(source_orientation); + + let source_to_listener = normalize_zero(source_position - listener_position); + // Angle between the source orientation vector and the source-listener vector + let angle = 180. * source_to_listener.dot(normalized_source_orientation).acos() / PI; + let abs_angle = angle.abs() as f64; + + // Divide by 2 here since API is entire angle (not half-angle) + let abs_inner_angle = self.cone_inner_angle.abs() / 2.; + let abs_outer_angle = self.cone_outer_angle.abs() / 2.; + + if abs_angle < abs_inner_angle { + // no attenuation + 1. + } else if abs_angle >= abs_outer_angle { + // max attenuation + self.cone_outer_gain + } else { + // gain changes linearly from 1 to cone_outer_gain + // as we go from inner to outer + let x = (abs_angle - abs_inner_angle) / (abs_outer_angle - abs_inner_angle); + (1. - x) + self.cone_outer_gain * x + } + } + + fn linear_distance(&self, mut distance: f64, rolloff_factor: f64) -> f64 { + if distance > self.max_distance { + distance = self.max_distance; + } + if distance < self.ref_distance { + distance = self.ref_distance; + } + let denom = self.max_distance - self.ref_distance; + 1. - rolloff_factor * (distance - self.ref_distance) / denom + } + + fn inverse_distance(&self, mut distance: f64, rolloff_factor: f64) -> f64 { + if distance < self.ref_distance { + distance = self.ref_distance; + } + let denom = self.ref_distance + rolloff_factor * (distance - self.ref_distance); + self.ref_distance / denom + } + + fn exponential_distance(&self, mut distance: f64, rolloff_factor: f64) -> f64 { + if distance < self.ref_distance { + distance = self.ref_distance; + } + + (distance / self.ref_distance).powf(-rolloff_factor) + } + + fn distance_gain_fn(&self) -> fn(&Self, f64, f64) -> f64 { + match self.distance_model { + DistanceModel::Linear => |x, d, r| x.linear_distance(d, r), + DistanceModel::Inverse => |x, d, r| x.inverse_distance(d, r), + DistanceModel::Exponential => |x, d, r| x.exponential_distance(d, r), + } + } +} + +impl AudioNodeEngine for PannerNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::PannerNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + let listener_data = if let Some(listener_data) = self.listener_data.take() { + listener_data + } else { + return inputs; + }; + + // We clamp this early + let rolloff_factor = + if self.distance_model == DistanceModel::Linear && self.rolloff_factor > 1. { + 1. + } else { + self.rolloff_factor + }; + + { + let block = &mut inputs.blocks[0]; + + block.explicit_repeat(); + + let mono = if block.chan_count() == 1 { + block.resize_silence(2); + true + } else { + debug_assert!(block.chan_count() == 2); + false + }; + + let distance_gain_fn = self.distance_gain_fn(); + + if self.panning_model == PanningModel::EqualPower { + let (l, r) = block.data_mut().split_at_mut(FRAMES_PER_BLOCK.0 as usize); + for frame in 0..FRAMES_PER_BLOCK.0 { + let frame = Tick(frame); + self.update_parameters(info, frame); + let data = listener_data.listener_data(frame); + let (mut azimuth, _elev, dist) = self.azimuth_elevation_distance(data); + let distance_gain = distance_gain_fn(self, dist, rolloff_factor); + let cone_gain = self.cone_gain(data); + + // https://webaudio.github.io/web-audio-api/#Spatialization-equal-power-panning + + // clamp to [-180, 180], then wrap to [-90, 90] + azimuth = azimuth.clamp(-180., 180.); + if azimuth < -90. { + azimuth = -180. - azimuth; + } else if azimuth > 90. { + azimuth = 180. - azimuth; + } + + let x = if mono { + (azimuth + 90.) / 180. + } else if azimuth <= 0. { + (azimuth + 90.) / 90. + } else { + azimuth / 90. + }; + let x = x * PI / 2.; + + let mut gain_l = x.cos(); + let mut gain_r = x.sin(); + // 9. * PI / 2 is often slightly negative, clamp + if gain_l <= 0. { + gain_l = 0. + } + if gain_r <= 0. { + gain_r = 0.; + } + + let index = frame.0 as usize; + if mono { + let input = l[index]; + l[index] = input * gain_l; + r[index] = input * gain_r; + } else if azimuth <= 0. { + l[index] += r[index] * gain_l; + r[index] *= gain_r; + } else { + r[index] += l[index] * gain_r; + l[index] *= gain_l; + } + l[index] = l[index] * distance_gain as f32 * cone_gain as f32; + r[index] = r[index] * distance_gain as f32 * cone_gain as f32; + } + } + } + + inputs + } + + fn input_count(&self) -> u32 { + 1 + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::Position(ParamDir::X) => &mut self.position_x, + ParamType::Position(ParamDir::Y) => &mut self.position_y, + ParamType::Position(ParamDir::Z) => &mut self.position_z, + ParamType::Orientation(ParamDir::X) => &mut self.orientation_x, + ParamType::Orientation(ParamDir::Y) => &mut self.orientation_y, + ParamType::Orientation(ParamDir::Z) => &mut self.orientation_z, + _ => panic!("Unknown param {:?} for PannerNode", id), + } + } + + fn set_listenerdata(&mut self, data: Block) { + self.listener_data = Some(data); + } + + fn message_specific(&mut self, message: AudioNodeMessage, _sample_rate: f32) { + if let AudioNodeMessage::PannerNode(p) = message { + match p { + PannerNodeMessage::SetPanningModel(p) => { + if p == PanningModel::HRTF { + log::warn!("HRTF requested but not supported"); + } + self.panning_model = p; + }, + PannerNodeMessage::SetDistanceModel(d) => self.distance_model = d, + PannerNodeMessage::SetRefDistance(val) => self.ref_distance = val, + PannerNodeMessage::SetMaxDistance(val) => self.max_distance = val, + PannerNodeMessage::SetRolloff(val) => self.rolloff_factor = val, + PannerNodeMessage::SetConeInner(val) => self.cone_inner_angle = val, + PannerNodeMessage::SetConeOuter(val) => self.cone_outer_angle = val, + PannerNodeMessage::SetConeGain(val) => self.cone_outer_gain = val, + } + } + } +} diff --git a/components/media/audio/param.rs b/components/media/audio/param.rs new file mode 100644 index 00000000000..54cf8d0e87e --- /dev/null +++ b/components/media/audio/param.rs @@ -0,0 +1,464 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::block::{Block, FRAMES_PER_BLOCK_USIZE, Tick}; +use crate::node::BlockInfo; + +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] +pub enum ParamType { + Frequency, + Detune, + Gain, + Q, + Pan, + PlaybackRate, + Position(ParamDir), + Forward(ParamDir), + Up(ParamDir), + Orientation(ParamDir), + Offset, +} + +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] +pub enum ParamDir { + X, + Y, + Z, +} + +/// An AudioParam. +/// +/// +pub struct Param { + val: f32, + kind: ParamRate, + events: Vec, + current_event: usize, + event_start_time: Tick, + event_start_value: f32, + /// Cache of inputs from connect()ed nodes + blocks: Vec, + /// The value of all connect()ed inputs mixed together, for this frame + block_mix_val: f32, + /// If true, `blocks` has been summed together into a single block + summed: bool, + dirty: bool, +} + +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum ParamRate { + /// Value is held for entire block + KRate, + /// Value is updated each frame + ARate, +} + +impl Param { + pub fn new(val: f32) -> Self { + Param { + val, + kind: ParamRate::ARate, + events: vec![], + current_event: 0, + event_start_time: Tick(0), + event_start_value: val, + blocks: Vec::new(), + block_mix_val: 0., + summed: false, + dirty: false, + } + } + + pub fn new_krate(val: f32) -> Self { + Param { + val, + kind: ParamRate::KRate, + events: vec![], + current_event: 0, + event_start_time: Tick(0), + event_start_value: val, + blocks: Vec::new(), + block_mix_val: 0., + summed: false, + dirty: false, + } + } + + /// Update the value of this param to the next + /// + /// Invariant: This should be called with monotonically increasing + /// ticks, and Tick(0) should never be skipped. + /// + /// Returns true if anything changed + pub fn update(&mut self, block: &BlockInfo, tick: Tick) -> bool { + let mut changed = self.dirty; + self.dirty = false; + if tick.0 == 0 { + self.summed = true; + if let Some(first) = self.blocks.pop() { + // first sum them together + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + let block = self + .blocks + .drain(..) + .fold(first, |acc, block| acc.sum(block)); + self.blocks.push(block); + } + } else if self.kind == ParamRate::KRate { + return changed; + } + + // Even if the timeline does nothing, it's still possible + // that there were connected inputs, so we should not + // directly return `false` after this point, instead returning + // `changed` + changed |= if let Some(block) = self.blocks.first() { + // store to be summed with `val` later + self.block_mix_val = block.data_chan_frame(tick.0 as usize, 0); + true + } else { + false + }; + + if self.events.len() <= self.current_event { + return changed; + } + + let current_tick = block.absolute_tick(tick); + let mut current_event = &self.events[self.current_event]; + + // move to next event if necessary + // XXXManishearth k-rate events may get skipped over completely by this + // method. Firefox currently doesn't support these, however, so we can + // handle those later + loop { + let mut move_next = false; + if let Some(done_time) = current_event.done_time() { + // If this event is done, move on + if done_time < current_tick { + move_next = true; + } + } else if let Some(next) = self.events.get(self.current_event + 1) { + // this event has no done time and we must run it till the next one + // starts + if let Some(start_time) = next.start_time() { + // if the next one is ready to start, move on + if start_time <= current_tick { + move_next = true; + } + } else { + // If we have a next event with no start time and + // the current event has no done time, this *has* to be because + // the current event is SetTargetAtTime and the next is a Ramp + // event. In this case we skip directly to the ramp assuming + // the SetTarget is ready to start (or has started already) + if current_event.time() <= current_tick { + move_next = true; + } else { + // This is a SetTarget event before its start time, ignore + return changed; + } + } + } + if move_next { + self.current_event += 1; + self.event_start_value = self.val; + self.event_start_time = current_tick; + if let Some(next) = self.events.get(self.current_event + 1) { + current_event = next; + // may need to move multiple times + continue; + } else { + return changed; + } + } + break; + } + + current_event.run( + &mut self.val, + current_tick, + self.event_start_time, + self.event_start_value, + ) + } + + pub fn value(&self) -> f32 { + // the data from connect()ed audionodes is first mixed + // together in update(), and then mixed with the actual param value + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + self.val + self.block_mix_val + } + + pub fn set_rate(&mut self, rate: ParamRate) { + self.kind = rate; + } + + pub(crate) fn insert_event(&mut self, event: AutomationEvent) { + if let AutomationEvent::SetValue(val) = event { + self.val = val; + self.event_start_value = val; + self.dirty = true; + return; + } + + let time = event.time(); + + let result = self.events.binary_search_by(|e| e.time().cmp(&time)); + // XXXManishearth this should handle overlapping events + let idx = match result { + Ok(idx) => idx, + Err(idx) => idx, + }; + + // XXXManishearth this isn't quite correct, this + // doesn't handle cases for when this lands inside a running + // event + if let Some(is_hold) = event.cancel_event() { + self.events.truncate(idx); + if !is_hold { + // If we cancelled the current event, reset + // the value to what it was before + if self.current_event >= self.events.len() { + self.val = self.event_start_value; + } + } + // don't actually insert the event + return; + } + self.events.insert(idx, event); + // XXXManishearth handle inserting events with a time before that + // of the current one + } + + pub(crate) fn add_block(&mut self, block: Block) { + debug_assert!(block.chan_count() == 1); + // summed only becomes true during a node's process() call, + // but add_block is called during graph traversal before processing, + // so if summed is true that means we've moved on to the next block + // and should clear our inputs + if self.summed { + self.blocks.clear(); + } + self.blocks.push(block) + } + + /// Flush an entire block of values into a buffer + /// + /// Only for use with AudioListener. + /// + /// Invariant: `block` must be a FRAMES_PER_BLOCK length array filled with silence + pub(crate) fn flush_to_block(&mut self, info: &BlockInfo, block: &mut [f32]) { + // common case + if self.current_event >= self.events.len() && self.blocks.is_empty() { + if self.val != 0. { + for block_tick in &mut block[0..FRAMES_PER_BLOCK_USIZE] { + // ideally this can use some kind of vectorized memset() + *block_tick = self.val; + } + } + // if the value is zero, our buffer is already zeroed + } else { + for block_tick in &mut block[0..FRAMES_PER_BLOCK_USIZE] { + self.update(info, Tick(*block_tick as u64)); + *block_tick = self.val; + } + } + } +} + +#[derive(Clone, Copy, Eq, PartialEq, Debug)] +pub enum RampKind { + Linear, + Exponential, +} + +#[derive(Clone, PartialEq, Debug)] +/// +pub(crate) enum AutomationEvent { + SetValue(f32), + SetValueAtTime(f32, Tick), + RampToValueAtTime(RampKind, f32, Tick), + SetTargetAtTime(f32, Tick, /* time constant, units of Tick */ f64), + SetValueCurveAtTime( + Vec, + /* start time */ Tick, + /* duration */ Tick, + ), + CancelAndHoldAtTime(Tick), + CancelScheduledValues(Tick), +} + +#[derive(Clone, PartialEq, Debug)] +/// An AutomationEvent that uses times in s instead of Ticks +pub enum UserAutomationEvent { + SetValue(f32), + SetValueAtTime(f32, /* time */ f64), + RampToValueAtTime(RampKind, f32, /* time */ f64), + SetTargetAtTime(f32, f64, /* time constant, units of s */ f64), + SetValueCurveAtTime(Vec, /* start time */ f64, /* duration */ f64), + CancelAndHoldAtTime(f64), + CancelScheduledValues(f64), +} + +impl UserAutomationEvent { + pub(crate) fn convert_to_event(self, rate: f32) -> AutomationEvent { + match self { + UserAutomationEvent::SetValue(val) => AutomationEvent::SetValue(val), + UserAutomationEvent::SetValueAtTime(val, time) => { + AutomationEvent::SetValueAtTime(val, Tick::from_time(time, rate)) + }, + UserAutomationEvent::RampToValueAtTime(kind, val, time) => { + AutomationEvent::RampToValueAtTime(kind, val, Tick::from_time(time, rate)) + }, + UserAutomationEvent::SetValueCurveAtTime(values, start, duration) => { + AutomationEvent::SetValueCurveAtTime( + values, + Tick::from_time(start, rate), + Tick::from_time(duration, rate), + ) + }, + UserAutomationEvent::SetTargetAtTime(val, start, tau) => { + AutomationEvent::SetTargetAtTime( + val, + Tick::from_time(start, rate), + tau * rate as f64, + ) + }, + UserAutomationEvent::CancelScheduledValues(t) => { + AutomationEvent::CancelScheduledValues(Tick::from_time(t, rate)) + }, + UserAutomationEvent::CancelAndHoldAtTime(t) => { + AutomationEvent::CancelAndHoldAtTime(Tick::from_time(t, rate)) + }, + } + } +} + +impl AutomationEvent { + /// The time of the event used for ordering + pub fn time(&self) -> Tick { + match *self { + AutomationEvent::SetValueAtTime(_, tick) => tick, + AutomationEvent::SetValueCurveAtTime(_, start, _) => start, + AutomationEvent::RampToValueAtTime(_, _, tick) => tick, + AutomationEvent::SetTargetAtTime(_, start, _) => start, + AutomationEvent::CancelAndHoldAtTime(t) => t, + AutomationEvent::CancelScheduledValues(tick) => tick, + AutomationEvent::SetValue(..) => { + unreachable!("SetValue should never appear in the timeline") + }, + } + } + + pub fn done_time(&self) -> Option { + match *self { + AutomationEvent::SetValueAtTime(_, tick) => Some(tick), + AutomationEvent::RampToValueAtTime(_, _, tick) => Some(tick), + AutomationEvent::SetValueCurveAtTime(_, start, duration) => Some(start + duration), + AutomationEvent::SetTargetAtTime(..) => None, + AutomationEvent::CancelAndHoldAtTime(t) => Some(t), + AutomationEvent::CancelScheduledValues(..) | AutomationEvent::SetValue(..) => { + unreachable!("CancelScheduledValues/SetValue should never appear in the timeline") + }, + } + } + + pub fn start_time(&self) -> Option { + match *self { + AutomationEvent::SetValueAtTime(_, tick) => Some(tick), + AutomationEvent::RampToValueAtTime(..) => None, + AutomationEvent::SetValueCurveAtTime(_, start, _) => Some(start), + AutomationEvent::SetTargetAtTime(_, start, _) => Some(start), + AutomationEvent::CancelAndHoldAtTime(t) => Some(t), + AutomationEvent::CancelScheduledValues(..) | AutomationEvent::SetValue(..) => { + unreachable!("CancelScheduledValues/SetValue should never appear in the timeline") + }, + } + } + + /// Returns Some if it's a cancel event + /// the boolean is if it's CancelAndHold + pub fn cancel_event(&self) -> Option { + match *self { + AutomationEvent::CancelAndHoldAtTime(..) => Some(true), + AutomationEvent::CancelScheduledValues(..) => Some(false), + _ => None, + } + } + + /// Update a parameter based on this event + /// + /// Returns true if something changed + pub fn run( + &self, + value: &mut f32, + current_tick: Tick, + event_start_time: Tick, + event_start_value: f32, + ) -> bool { + if matches!(self.start_time(), Some(start_time) if start_time > current_tick) { + // The previous event finished and we advanced to this + // event, but it's not started yet. Return early + return false; + } + + match *self { + AutomationEvent::SetValueAtTime(val, time) => { + if current_tick == time { + *value = val; + true + } else { + false + } + }, + AutomationEvent::RampToValueAtTime(kind, val, time) => { + let progress = + (current_tick - event_start_time).0 as f32 / (time - event_start_time).0 as f32; + match kind { + RampKind::Linear => { + *value = event_start_value + (val - event_start_value) * progress; + }, + RampKind::Exponential => { + let ratio = val / event_start_value; + if event_start_value == 0. || ratio < 0. { + if time == current_tick { + *value = val; + } else { + *value = event_start_value; + } + } else { + *value = event_start_value * (ratio).powf(progress); + } + }, + } + true + }, + AutomationEvent::SetTargetAtTime(val, start, tau) => { + let exp = -((current_tick - start) / tau); + *value = val + (event_start_value - val) * exp.exp() as f32; + true + }, + AutomationEvent::SetValueCurveAtTime(ref values, start, duration) => { + let progress = ((current_tick.0 as f32) - (start.0 as f32)) / (duration.0 as f32); + debug_assert!(progress >= 0.); + let n = values.len() as f32; + let k_float = (n - 1.) * progress; + let k = k_float.floor(); + if (k + 1.) < n { + let progress = k_float - k; + *value = + values[k as usize] * (1. - progress) + values[(k + 1.) as usize] * progress; + } else { + *value = values[(n - 1.) as usize]; + } + true + }, + AutomationEvent::CancelAndHoldAtTime(..) => false, + AutomationEvent::CancelScheduledValues(..) | AutomationEvent::SetValue(..) => { + unreachable!("CancelScheduledValues/SetValue should never appear in the timeline") + }, + } + } +} diff --git a/components/media/audio/render_thread.rs b/components/media/audio/render_thread.rs new file mode 100644 index 00000000000..a294cc1d60d --- /dev/null +++ b/components/media/audio/render_thread.rs @@ -0,0 +1,376 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::mpsc::{Receiver, Sender}; + +use servo_media_streams::{MediaSocket, MediaStreamId}; + +use crate::analyser_node::AnalyserNode; +use crate::biquad_filter_node::BiquadFilterNode; +use crate::block::{Chunk, FRAMES_PER_BLOCK, Tick}; +use crate::buffer_source_node::AudioBufferSourceNode; +use crate::channel_node::{ChannelMergerNode, ChannelSplitterNode}; +use crate::constant_source_node::ConstantSourceNode; +use crate::context::{AudioContextOptions, ProcessingState, StateChangeResult}; +use crate::gain_node::GainNode; +use crate::graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; +use crate::iir_filter_node::IIRFilterNode; +use crate::media_element_source_node::MediaElementSourceNode; +use crate::media_stream_destination_node::MediaStreamDestinationNode; +use crate::media_stream_source_node::MediaStreamSourceNode; +use crate::node::{AudioNodeEngine, AudioNodeInit, AudioNodeMessage, BlockInfo, ChannelInfo}; +use crate::offline_sink::OfflineAudioSink; +use crate::oscillator_node::OscillatorNode; +use crate::panner_node::PannerNode; +use crate::sink::{AudioSink, AudioSinkError}; +use crate::stereo_panner::StereoPannerNode; +use crate::wave_shaper_node::WaveShaperNode; +use crate::{AudioBackend, AudioStreamReader}; + +pub type SinkEosCallback = Box>) + Send + Sync + 'static>; + +pub enum AudioRenderThreadMsg { + CreateNode(AudioNodeInit, Sender, ChannelInfo), + ConnectPorts(PortId, PortId), + MessageNode(NodeId, AudioNodeMessage), + Resume(Sender), + Suspend(Sender), + Close(Sender), + SinkNeedData, + GetCurrentTime(Sender), + + DisconnectAllFrom(NodeId), + DisconnectOutput(PortId), + DisconnectBetween(NodeId, NodeId), + DisconnectTo(NodeId, PortId), + DisconnectOutputBetween(PortId, NodeId), + DisconnectOutputBetweenTo(PortId, PortId), + + SetSinkEosCallback(SinkEosCallback), + + SetMute(bool), +} + +pub enum Sink { + RealTime(Box), + Offline(OfflineAudioSink), +} + +impl AudioSink for Sink { + fn init( + &self, + sample_rate: f32, + sender: Sender, + ) -> Result<(), AudioSinkError> { + match *self { + Sink::RealTime(ref sink) => sink.init(sample_rate, sender), + Sink::Offline(ref sink) => { + sink.init(sample_rate, sender).unwrap(); + Ok(()) + }, + } + } + + fn init_stream(&self, _: u8, _: f32, _: Box) -> Result<(), AudioSinkError> { + unreachable!("Sink should never be used for MediaStreamDestinationNode") + } + + fn play(&self) -> Result<(), AudioSinkError> { + match *self { + Sink::RealTime(ref sink) => sink.play(), + Sink::Offline(ref sink) => { + sink.play().unwrap(); + Ok(()) + }, + } + } + + fn stop(&self) -> Result<(), AudioSinkError> { + match *self { + Sink::RealTime(ref sink) => sink.stop(), + Sink::Offline(ref sink) => { + sink.stop().unwrap(); + Ok(()) + }, + } + } + + fn has_enough_data(&self) -> bool { + match *self { + Sink::RealTime(ref sink) => sink.has_enough_data(), + Sink::Offline(ref sink) => sink.has_enough_data(), + } + } + + fn push_data(&self, chunk: Chunk) -> Result<(), AudioSinkError> { + match *self { + Sink::RealTime(ref sink) => sink.push_data(chunk), + Sink::Offline(ref sink) => { + sink.push_data(chunk).unwrap(); + Ok(()) + }, + } + } + + fn set_eos_callback( + &self, + callback: Box>) + Send + Sync + 'static>, + ) { + match *self { + Sink::RealTime(ref sink) => sink.set_eos_callback(callback), + Sink::Offline(ref sink) => sink.set_eos_callback(callback), + } + } +} + +pub struct AudioRenderThread { + pub graph: AudioGraph, + pub sink: Sink, + pub sink_factory: Box Result, AudioSinkError>>, + pub reader_factory: Box Box>, + pub state: ProcessingState, + pub sample_rate: f32, + pub current_time: f64, + pub current_frame: Tick, + pub muted: bool, +} + +impl AudioRenderThread { + /// Initializes the AudioRenderThread object + /// + /// You must call .event_loop() on this to run it! + fn prepare_thread( + sender: Sender, + sample_rate: f32, + graph: AudioGraph, + options: AudioContextOptions, + ) -> Result { + let sink_factory = Box::new(|| B::make_sink().map(|s| Box::new(s) as Box)); + let reader_factory = Box::new(|id, sample_rate| B::make_streamreader(id, sample_rate)); + let sink = match options { + AudioContextOptions::RealTimeAudioContext(_) => Sink::RealTime(sink_factory()?), + AudioContextOptions::OfflineAudioContext(options) => Sink::Offline( + OfflineAudioSink::new(options.channels as usize, options.length), + ), + }; + + sink.init(sample_rate, sender)?; + + Ok(Self { + graph, + sink, + sink_factory, + reader_factory, + state: ProcessingState::Suspended, + sample_rate, + current_time: 0., + current_frame: Tick(0), + muted: false, + }) + } + + /// Start the audio render thread + /// + /// In case something fails, it will instead start a thread with a dummy backend + pub fn start( + event_queue: Receiver, + sender: Sender, + sample_rate: f32, + graph: AudioGraph, + options: AudioContextOptions, + init_sender: Sender>, + ) { + let mut thread = + match Self::prepare_thread::(sender.clone(), sample_rate, graph, options) { + Ok(thread) => { + let _ = init_sender.send(Ok(())); + thread + }, + Err(e) => { + let _ = init_sender.send(Err(e)); + return; + }, + }; + + thread.event_loop(event_queue); + } + + make_render_thread_state_change!(resume, Running, play); + + make_render_thread_state_change!(suspend, Suspended, stop); + + fn create_node(&mut self, node_type: AudioNodeInit, ch: ChannelInfo) -> NodeId { + let mut needs_listener = false; + let mut is_dest = false; + let node: Box = match node_type { + AudioNodeInit::AnalyserNode(sender) => Box::new(AnalyserNode::new(sender, ch)), + AudioNodeInit::AudioBufferSourceNode(options) => { + Box::new(AudioBufferSourceNode::new(options, ch)) + }, + AudioNodeInit::BiquadFilterNode(options) => { + Box::new(BiquadFilterNode::new(options, ch, self.sample_rate)) + }, + AudioNodeInit::GainNode(options) => Box::new(GainNode::new(options, ch)), + AudioNodeInit::StereoPannerNode(options) => { + Box::new(StereoPannerNode::new(options, ch)) + }, + AudioNodeInit::PannerNode(options) => { + needs_listener = true; + Box::new(PannerNode::new(options, ch)) + }, + AudioNodeInit::MediaStreamSourceNode(id) => { + let reader = (self.reader_factory)(id, self.sample_rate); + Box::new(MediaStreamSourceNode::new(reader, ch)) + }, + AudioNodeInit::OscillatorNode(options) => Box::new(OscillatorNode::new(options, ch)), + AudioNodeInit::ChannelMergerNode(options) => { + Box::new(ChannelMergerNode::new(options, ch)) + }, + AudioNodeInit::ConstantSourceNode(options) => { + Box::new(ConstantSourceNode::new(options, ch)) + }, + AudioNodeInit::MediaStreamDestinationNode(socket) => { + is_dest = true; + Box::new(MediaStreamDestinationNode::new( + socket, + self.sample_rate, + (self.sink_factory)().unwrap(), + ch, + )) + }, + AudioNodeInit::ChannelSplitterNode => Box::new(ChannelSplitterNode::new(ch)), + AudioNodeInit::WaveShaperNode(options) => Box::new(WaveShaperNode::new(options, ch)), + AudioNodeInit::MediaElementSourceNode => Box::new(MediaElementSourceNode::new(ch)), + AudioNodeInit::IIRFilterNode(options) => Box::new(IIRFilterNode::new(options, ch)), + _ => unimplemented!(), + }; + let id = self.graph.add_node(node); + if needs_listener { + let listener = self.graph.listener_id().output(0); + self.graph.add_edge(listener, id.listener()); + } + if is_dest { + self.graph.add_extra_dest(id); + } + id + } + + fn connect_ports(&mut self, output: PortId, input: PortId) { + self.graph.add_edge(output, input) + } + + fn process(&mut self) -> Chunk { + if self.muted { + return Chunk::explicit_silence(); + } + + let info = BlockInfo { + sample_rate: self.sample_rate, + frame: self.current_frame, + time: self.current_time, + }; + self.graph.process(&info) + } + + fn set_mute(&mut self, val: bool) { + self.muted = val; + } + + fn event_loop(&mut self, event_queue: Receiver) { + let sample_rate = self.sample_rate; + let handle_msg = move |context: &mut Self, msg: AudioRenderThreadMsg| -> bool { + let mut break_loop = false; + match msg { + AudioRenderThreadMsg::CreateNode(node_type, tx, ch) => { + let _ = tx.send(context.create_node(node_type, ch)); + }, + AudioRenderThreadMsg::ConnectPorts(output, input) => { + context.connect_ports(output, input); + }, + AudioRenderThreadMsg::Resume(tx) => { + let _ = tx.send(context.resume()); + }, + AudioRenderThreadMsg::Suspend(tx) => { + let _ = tx.send(context.suspend()); + }, + AudioRenderThreadMsg::Close(tx) => { + let _ = tx.send(context.suspend()); + break_loop = true; + }, + AudioRenderThreadMsg::GetCurrentTime(response) => { + response.send(context.current_time).unwrap() + }, + AudioRenderThreadMsg::MessageNode(id, msg) => { + context.graph.node_mut(id).message(msg, sample_rate) + }, + AudioRenderThreadMsg::SinkNeedData => { + // Do nothing. This will simply unblock the thread so we + // can restart the non-blocking event loop. + }, + AudioRenderThreadMsg::DisconnectAllFrom(id) => { + context.graph.disconnect_all_from(id) + }, + AudioRenderThreadMsg::DisconnectOutput(out) => context.graph.disconnect_output(out), + AudioRenderThreadMsg::DisconnectBetween(from, to) => { + context.graph.disconnect_between(from, to) + }, + AudioRenderThreadMsg::DisconnectTo(from, to) => { + context.graph.disconnect_to(from, to) + }, + AudioRenderThreadMsg::DisconnectOutputBetween(from, to) => { + context.graph.disconnect_output_between(from, to) + }, + AudioRenderThreadMsg::DisconnectOutputBetweenTo(from, to) => { + context.graph.disconnect_output_between_to(from, to) + }, + AudioRenderThreadMsg::SetSinkEosCallback(callback) => { + context.sink.set_eos_callback(callback); + }, + AudioRenderThreadMsg::SetMute(val) => { + context.set_mute(val); + }, + }; + + break_loop + }; + + loop { + if self.sink.has_enough_data() || self.state == ProcessingState::Suspended { + // If we are not processing audio or + // if we have already pushed enough data into the audio sink + // we wait for messages coming from the control thread or + // the audio sink. The audio sink will notify whenever it + // needs more data. + if event_queue.recv().is_ok_and(|msg| handle_msg(self, msg)) { + break; + } + } else { + // If we have not pushed enough data into the audio sink yet, + // we process the control message queue + if event_queue + .try_recv() + .is_ok_and(|msg| handle_msg(self, msg)) + { + break; + } + + if self.state == ProcessingState::Suspended { + // Bail out if we just suspended processing. + continue; + } + + // push into the audio sink the result of processing a + // render quantum. + let data = self.process(); + if self.sink.push_data(data).is_ok() { + // increment current frame by the render quantum size. + self.current_frame += FRAMES_PER_BLOCK; + self.current_time = self.current_frame / self.sample_rate as f64; + } else { + eprintln!("Could not push data to audio sink"); + } + } + } + } +} diff --git a/components/media/audio/sink.rs b/components/media/audio/sink.rs new file mode 100644 index 00000000000..234b81dc855 --- /dev/null +++ b/components/media/audio/sink.rs @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::mpsc::Sender; + +use servo_media_streams::MediaSocket; + +use crate::block::Chunk; +use crate::render_thread::{AudioRenderThreadMsg, SinkEosCallback}; + +#[derive(Debug, PartialEq)] +pub enum AudioSinkError { + /// Backend specific error. + Backend(String), + /// Could not push buffer into the audio sink. + BufferPushFailed, + /// Could not move to a different state. + StateChangeFailed, +} + +pub trait AudioSink: Send { + fn init( + &self, + sample_rate: f32, + render_thread_channel: Sender, + ) -> Result<(), AudioSinkError>; + fn init_stream( + &self, + channels: u8, + sample_rate: f32, + socket: Box, + ) -> Result<(), AudioSinkError>; + fn play(&self) -> Result<(), AudioSinkError>; + fn stop(&self) -> Result<(), AudioSinkError>; + fn has_enough_data(&self) -> bool; + fn push_data(&self, chunk: Chunk) -> Result<(), AudioSinkError>; + fn set_eos_callback(&self, callback: SinkEosCallback); +} diff --git a/components/media/audio/stereo_panner.rs b/components/media/audio/stereo_panner.rs new file mode 100644 index 00000000000..dc3c6fb8085 --- /dev/null +++ b/components/media/audio/stereo_panner.rs @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::f32::consts::PI; + +use crate::block::{Chunk, FRAMES_PER_BLOCK, Tick}; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; +use crate::param::{Param, ParamType}; + +#[derive(Copy, Clone, Debug)] +pub struct StereoPannerOptions { + pub pan: f32, +} + +impl Default for StereoPannerOptions { + fn default() -> Self { + StereoPannerOptions { pan: 0. } + } +} + +#[derive(AudioNodeCommon)] +pub(crate) struct StereoPannerNode { + channel_info: ChannelInfo, + pan: Param, +} + +impl StereoPannerNode { + pub fn new(options: StereoPannerOptions, channel_info: ChannelInfo) -> Self { + Self { + channel_info, + pan: Param::new(options.pan), + } + } + + pub fn update_parameters(&mut self, info: &BlockInfo, tick: Tick) -> bool { + self.pan.update(info, tick) + } +} + +impl AudioNodeEngine for StereoPannerNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::StereoPannerNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + { + let block = &mut inputs.blocks[0]; + + block.explicit_repeat(); + + let mono = if block.chan_count() == 1 { + block.resize_silence(2); + true + } else { + debug_assert!(block.chan_count() == 2); + false + }; + + let (l, r) = block.data_mut().split_at_mut(FRAMES_PER_BLOCK.0 as usize); + let mut pan = self.pan.value(); + for frame in 0..FRAMES_PER_BLOCK.0 { + let frame = Tick(frame); + if self.update_parameters(info, frame) { + pan = self.pan.value(); + } + + // https://webaudio.github.io/web-audio-api/#stereopanner-algorithm + + // clamp pan to [-1, 1] + pan = pan.clamp(-1., 1.); + + let x = if mono { + (pan + 1.) / 2. + } else if pan <= 0. { + pan + 1. + } else { + pan + }; + let x = x * PI / 2.; + + let mut gain_l = x.cos(); + let mut gain_r = x.sin(); + // 9. * PI / 2 is often slightly negative, clamp + if gain_l <= 0. { + gain_l = 0. + } + if gain_r <= 0. { + gain_r = 0.; + } + + let index = frame.0 as usize; + if mono { + let input = l[index]; + l[index] = input * gain_l; + r[index] = input * gain_r; + } else if pan <= 0. { + l[index] += r[index] * gain_l; + r[index] *= gain_r; + } else { + r[index] += l[index] * gain_r; + l[index] *= gain_l; + } + } + } + + inputs + } + + fn input_count(&self) -> u32 { + 1 + } + + fn get_param(&mut self, id: ParamType) -> &mut Param { + match id { + ParamType::Pan => &mut self.pan, + _ => panic!("Unknown param {:?} for PannerNode", id), + } + } +} diff --git a/components/media/audio/wave_shaper_node.rs b/components/media/audio/wave_shaper_node.rs new file mode 100644 index 00000000000..aac54043263 --- /dev/null +++ b/components/media/audio/wave_shaper_node.rs @@ -0,0 +1,255 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use speexdsp_resampler::State as SpeexResamplerState; + +use crate::block::{Chunk, FRAMES_PER_BLOCK_USIZE}; +use crate::node::{AudioNodeEngine, AudioNodeType, BlockInfo, ChannelInfo}; + +#[derive(Clone, Debug, PartialEq)] +pub enum OverSampleType { + None, + Double, + Quadruple, +} + +#[derive(Clone, Debug, PartialEq)] +enum TailtimeBlocks { + Zero, + One, + Two, +} + +const OVERSAMPLING_QUALITY: usize = 0; + +impl OverSampleType { + fn value(&self) -> usize { + match self { + OverSampleType::None => 1, + OverSampleType::Double => 2, + OverSampleType::Quadruple => 4, + } + } +} + +type WaveShaperCurve = Option>; + +#[derive(Clone, Debug)] +pub struct WaveShaperNodeOptions { + pub curve: WaveShaperCurve, + pub oversample: OverSampleType, +} + +impl Default for WaveShaperNodeOptions { + fn default() -> Self { + WaveShaperNodeOptions { + curve: None, + oversample: OverSampleType::None, + } + } +} + +#[derive(Clone, Debug)] +pub enum WaveShaperNodeMessage { + SetCurve(WaveShaperCurve), +} + +#[derive(AudioNodeCommon)] +pub(crate) struct WaveShaperNode { + curve_set: bool, + curve: WaveShaperCurve, + #[allow(dead_code)] + oversample: OverSampleType, + channel_info: ChannelInfo, + upsampler: Option, + downsampler: Option, + tailtime_blocks_left: TailtimeBlocks, +} + +impl WaveShaperNode { + pub fn new(options: WaveShaperNodeOptions, channel_info: ChannelInfo) -> Self { + if let Some(vec) = &options.curve { + assert!( + vec.len() > 1, + "WaveShaperNode curve must have length of 2 or more" + ) + } + + Self { + curve_set: options.curve.is_some(), + curve: options.curve, + oversample: options.oversample, + channel_info, + upsampler: None, + downsampler: None, + tailtime_blocks_left: TailtimeBlocks::Zero, + } + } + + fn handle_waveshaper_message(&mut self, message: WaveShaperNodeMessage, _sample_rate: f32) { + match message { + WaveShaperNodeMessage::SetCurve(new_curve) => { + if self.curve_set && new_curve.is_some() { + panic!("InvalidStateError: cant set curve if it was already set"); + } + self.curve_set = new_curve.is_some(); + self.curve = new_curve; + }, + } + } +} + +impl AudioNodeEngine for WaveShaperNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::WaveShaperNode + } + + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + if self.curve.is_none() { + return inputs; + } + + let curve = &self.curve.as_ref().expect("Just checked for is_none()"); + + if inputs.blocks[0].is_silence() { + if WaveShaperNode::silence_produces_nonsilent_output(curve) { + inputs.blocks[0].explicit_silence(); + self.tailtime_blocks_left = TailtimeBlocks::Two; + } else if self.tailtime_blocks_left != TailtimeBlocks::Zero { + inputs.blocks[0].explicit_silence(); + + self.tailtime_blocks_left = match self.tailtime_blocks_left { + TailtimeBlocks::Zero => TailtimeBlocks::Zero, + TailtimeBlocks::One => TailtimeBlocks::Zero, + TailtimeBlocks::Two => TailtimeBlocks::One, + } + } else { + return inputs; + } + } else { + self.tailtime_blocks_left = TailtimeBlocks::Two; + } + + let block = &mut inputs.blocks[0]; + let channels = block.chan_count(); + + if self.oversample != OverSampleType::None { + let rate: usize = info.sample_rate as usize; + let sampling_factor = self.oversample.value(); + + if self.upsampler.is_none() { + self.upsampler = Some( + SpeexResamplerState::new( + channels as usize, + rate, + rate * sampling_factor, + OVERSAMPLING_QUALITY, + ) + .expect("Couldnt create upsampler"), + ); + }; + + if self.downsampler.is_none() { + self.downsampler = Some( + SpeexResamplerState::new( + channels as usize, + rate * sampling_factor, + rate, + OVERSAMPLING_QUALITY, + ) + .expect("Couldnt create downsampler"), + ); + }; + + let upsampler = self.upsampler.as_mut().unwrap(); + let downsampler = self.downsampler.as_mut().unwrap(); + + let mut oversampled_buffer: Vec = + vec![0.; FRAMES_PER_BLOCK_USIZE * sampling_factor]; + + for chan in 0..channels { + let out_len = WaveShaperNode::resample( + upsampler, + chan, + block.data_chan(chan), + &mut oversampled_buffer, + ); + + debug_assert!( + out_len == 128 * sampling_factor, + "Expected {} samples in output after upsampling, got: {}", + 128 * sampling_factor, + out_len + ); + + WaveShaperNode::apply_curve(&mut oversampled_buffer, curve); + + let out_len = WaveShaperNode::resample( + downsampler, + chan, + &oversampled_buffer, + block.data_chan_mut(chan), + ); + + debug_assert!( + out_len == 128, + "Expected 128 samples in output after downsampling, got {}", + out_len + ); + } + } else { + WaveShaperNode::apply_curve(block.data_mut(), curve); + } + + inputs + } + + make_message_handler!(WaveShaperNode: handle_waveshaper_message); +} + +impl WaveShaperNode { + fn silence_produces_nonsilent_output(curve: &[f32]) -> bool { + let len = curve.len(); + let len_halved = ((len - 1) as f32) / 2.; + let curve_index: f32 = len_halved; + let index_lo = curve_index as usize; + let index_hi = index_lo + 1; + let interp_factor: f32 = curve_index - index_lo as f32; + let shaped_val = (1. - interp_factor) * curve[index_lo] + interp_factor * curve[index_hi]; + shaped_val == 0.0 + } + + fn apply_curve(buf: &mut [f32], curve: &[f32]) { + let len = curve.len(); + let len_halved = ((len - 1) as f32) / 2.; + buf.iter_mut().for_each(|sample| { + let curve_index: f32 = len_halved * (*sample + 1.); + + if curve_index <= 0. { + *sample = curve[0]; + } else if curve_index >= (len - 1) as f32 { + *sample = curve[len - 1]; + } else { + let index_lo = curve_index as usize; + let index_hi = index_lo + 1; + let interp_factor: f32 = curve_index - index_lo as f32; + *sample = (1. - interp_factor) * curve[index_lo] + interp_factor * curve[index_hi]; + } + }); + } + + fn resample( + st: &mut SpeexResamplerState, + chan: u8, + input: &[f32], + output: &mut [f32], + ) -> usize { + let (_in_len, out_len) = st + .process_float(chan as usize, input, output) + .expect("Resampling failed"); + out_len + } +} diff --git a/components/media/backends/auto/Cargo.toml b/components/media/backends/auto/Cargo.toml new file mode 100644 index 00000000000..63cb9f060a2 --- /dev/null +++ b/components/media/backends/auto/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "servo-media-auto" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_auto" +path = "lib.rs" + +[target.'cfg(any(all(target_os = "android", any(target_arch = "arm", target_arch = "aarch64")), target_arch = "x86_64",target_arch = "aarch64"))'.dependencies.servo-media-gstreamer] +path = "../gstreamer" + +[target.'cfg(not(any(all(target_os = "android", any(target_arch = "arm", target_arch = "aarch64")), target_arch = "x86_64", target_arch = "aarch64")))'.dependencies.servo-media-dummy] +path = "../dummy" diff --git a/components/media/backends/auto/lib.rs b/components/media/backends/auto/lib.rs new file mode 100644 index 00000000000..fdec73fa12c --- /dev/null +++ b/components/media/backends/auto/lib.rs @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#[cfg(any( + all( + target_os = "android", + any(target_arch = "arm", target_arch = "aarch64") + ), + target_arch = "x86_64", + target_arch = "aarch64", +))] +mod platform { + pub use servo_media_gstreamer::GStreamerBackend as Backend; +} + +#[cfg(not(any( + all( + target_os = "android", + any(target_arch = "arm", target_arch = "aarch64") + ), + target_arch = "x86_64", + target_arch = "aarch64", +)))] +mod platform { + pub use servo_media_dummy::DummyBackend as Backend; +} + +pub type Backend = platform::Backend; diff --git a/components/media/backends/dummy/Cargo.toml b/components/media/backends/dummy/Cargo.toml new file mode 100644 index 00000000000..d78db37efaf --- /dev/null +++ b/components/media/backends/dummy/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "servo-media-dummy" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_dummy" +path = "lib.rs" + +[dependencies] +ipc-channel = { workspace = true } +servo-media = { path = "../../servo-media" } +servo-media-audio = { path = "../../audio" } +servo-media-player = { path = "../../player" } +servo-media-streams = { path = "../../streams" } +servo-media-traits = { path = "../../traits" } +servo-media-webrtc = { path = "../../webrtc" } diff --git a/components/media/backends/dummy/lib.rs b/components/media/backends/dummy/lib.rs new file mode 100644 index 00000000000..8bcd34cb0f2 --- /dev/null +++ b/components/media/backends/dummy/lib.rs @@ -0,0 +1,393 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate ipc_channel; +extern crate servo_media; +extern crate servo_media_audio; +extern crate servo_media_player; +extern crate servo_media_streams; +extern crate servo_media_traits; +extern crate servo_media_webrtc; + +use std::any::Any; +use std::ops::Range; +use std::sync::mpsc::{self, Sender}; +use std::sync::{Arc, Mutex}; + +use ipc_channel::ipc::IpcSender; +use servo_media::{Backend, BackendInit, MediaInstanceError, SupportsMediaType}; +use servo_media_audio::block::{Block, Chunk}; +use servo_media_audio::context::{AudioContext, AudioContextOptions}; +use servo_media_audio::decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecoderOptions}; +use servo_media_audio::render_thread::AudioRenderThreadMsg; +use servo_media_audio::sink::{AudioSink, AudioSinkError}; +use servo_media_audio::{AudioBackend, AudioStreamReader}; +use servo_media_player::context::PlayerGLContext; +use servo_media_player::{Player, PlayerError, PlayerEvent, StreamType, audio, video}; +use servo_media_streams::capture::MediaTrackConstraintSet; +use servo_media_streams::device_monitor::{MediaDeviceInfo, MediaDeviceMonitor}; +use servo_media_streams::registry::{MediaStreamId, register_stream, unregister_stream}; +use servo_media_streams::{MediaOutput, MediaSocket, MediaStream, MediaStreamType}; +use servo_media_traits::{ClientContextId, MediaInstance}; +use servo_media_webrtc::{ + BundlePolicy, DataChannelId, DataChannelInit, DataChannelMessage, IceCandidate, + SessionDescription, WebRtcBackend, WebRtcController, WebRtcControllerBackend, + WebRtcDataChannelResult, WebRtcResult, WebRtcSignaller, thread, +}; + +pub struct DummyBackend; + +impl BackendInit for DummyBackend { + fn init() -> Box { + Box::new(DummyBackend) + } +} + +impl Backend for DummyBackend { + fn create_audiostream(&self) -> MediaStreamId { + register_stream(Arc::new(Mutex::new(DummyMediaStream { + id: MediaStreamId::new(), + }))) + } + + fn create_videostream(&self) -> MediaStreamId { + register_stream(Arc::new(Mutex::new(DummyMediaStream { + id: MediaStreamId::new(), + }))) + } + + fn create_stream_output(&self) -> Box { + Box::new(DummyMediaOutput) + } + + fn create_audioinput_stream(&self, _: MediaTrackConstraintSet) -> Option { + Some(register_stream(Arc::new(Mutex::new(DummyMediaStream { + id: MediaStreamId::new(), + })))) + } + + fn create_stream_and_socket( + &self, + _: MediaStreamType, + ) -> (Box, MediaStreamId) { + let id = register_stream(Arc::new(Mutex::new(DummyMediaStream { + id: MediaStreamId::new(), + }))); + (Box::new(DummySocket), id) + } + + fn create_videoinput_stream(&self, _: MediaTrackConstraintSet) -> Option { + Some(register_stream(Arc::new(Mutex::new(DummyMediaStream { + id: MediaStreamId::new(), + })))) + } + + fn create_player( + &self, + _id: &ClientContextId, + _: StreamType, + _: IpcSender, + _: Option>>, + _: Option>>, + _: Box, + ) -> Arc> { + Arc::new(Mutex::new(DummyPlayer)) + } + + fn create_audio_context( + &self, + _id: &ClientContextId, + options: AudioContextOptions, + ) -> Result>, AudioSinkError> { + let (sender, _) = mpsc::channel(); + let sender = Arc::new(Mutex::new(sender)); + Ok(Arc::new(Mutex::new(AudioContext::new::( + 0, + &ClientContextId::build(1, 1), + sender, + options, + )?))) + } + + fn create_webrtc(&self, signaller: Box) -> WebRtcController { + WebRtcController::new::(signaller) + } + + fn can_play_type(&self, _media_type: &str) -> SupportsMediaType { + SupportsMediaType::No + } + + fn get_device_monitor(&self) -> Box { + Box::new(DummyMediaDeviceMonitor {}) + } +} + +impl AudioBackend for DummyBackend { + type Sink = DummyAudioSink; + fn make_decoder() -> Box { + Box::new(DummyAudioDecoder) + } + + fn make_sink() -> Result { + Ok(DummyAudioSink) + } + fn make_streamreader( + _id: MediaStreamId, + _sample_rate: f32, + ) -> Box { + Box::new(DummyStreamReader) + } +} + +pub struct DummyPlayer; + +pub struct DummyStreamReader; + +impl AudioStreamReader for DummyStreamReader { + fn pull(&self) -> Block { + Default::default() + } + fn start(&self) {} + fn stop(&self) {} +} + +impl Player for DummyPlayer { + fn play(&self) -> Result<(), PlayerError> { + Ok(()) + } + fn pause(&self) -> Result<(), PlayerError> { + Ok(()) + } + + fn paused(&self) -> bool { + true + } + fn can_resume(&self) -> bool { + true + } + + fn stop(&self) -> Result<(), PlayerError> { + Ok(()) + } + fn seek(&self, _: f64) -> Result<(), PlayerError> { + Ok(()) + } + + fn set_mute(&self, _: bool) -> Result<(), PlayerError> { + Ok(()) + } + + fn muted(&self) -> bool { + false + } + + fn set_volume(&self, _: f64) -> Result<(), PlayerError> { + Ok(()) + } + + fn volume(&self) -> f64 { + 1.0 + } + + fn set_input_size(&self, _: u64) -> Result<(), PlayerError> { + Ok(()) + } + + fn set_playback_rate(&self, _: f64) -> Result<(), PlayerError> { + Ok(()) + } + + fn playback_rate(&self) -> f64 { + 1.0 + } + + fn push_data(&self, _: Vec) -> Result<(), PlayerError> { + Ok(()) + } + fn end_of_stream(&self) -> Result<(), PlayerError> { + Ok(()) + } + + fn buffered(&self) -> Vec> { + vec![] + } + + fn seekable(&self) -> Vec> { + vec![] + } + + fn set_stream(&self, _: &MediaStreamId, _: bool) -> Result<(), PlayerError> { + Ok(()) + } + + fn render_use_gl(&self) -> bool { + false + } + fn set_audio_track(&self, _: i32, _: bool) -> Result<(), PlayerError> { + Ok(()) + } + fn set_video_track(&self, _: i32, _: bool) -> Result<(), PlayerError> { + Ok(()) + } +} + +impl WebRtcBackend for DummyBackend { + type Controller = DummyWebRtcController; + fn construct_webrtc_controller( + _: Box, + _: WebRtcController, + ) -> Self::Controller { + DummyWebRtcController + } +} + +pub struct DummyAudioDecoder; + +impl AudioDecoder for DummyAudioDecoder { + fn decode(&self, _: Vec, _: AudioDecoderCallbacks, _: Option) {} +} + +pub struct DummySocket; + +impl MediaSocket for DummySocket { + fn as_any(&self) -> &dyn Any { + self + } +} + +pub struct DummyMediaStream { + id: MediaStreamId, +} + +impl MediaStream for DummyMediaStream { + fn as_any(&self) -> &dyn Any { + self + } + fn as_mut_any(&mut self) -> &mut dyn Any { + self + } + fn set_id(&mut self, _: MediaStreamId) {} + + fn ty(&self) -> MediaStreamType { + MediaStreamType::Audio + } +} + +impl Drop for DummyMediaStream { + fn drop(&mut self) { + unregister_stream(&self.id); + } +} + +pub struct DummyAudioSink; + +impl AudioSink for DummyAudioSink { + fn init(&self, _: f32, _: Sender) -> Result<(), AudioSinkError> { + Ok(()) + } + fn init_stream(&self, _: u8, _: f32, _: Box) -> Result<(), AudioSinkError> { + Ok(()) + } + fn play(&self) -> Result<(), AudioSinkError> { + Ok(()) + } + fn stop(&self) -> Result<(), AudioSinkError> { + Ok(()) + } + fn has_enough_data(&self) -> bool { + true + } + fn push_data(&self, _: Chunk) -> Result<(), AudioSinkError> { + Ok(()) + } + fn set_eos_callback(&self, _: Box>) + Send + Sync + 'static>) {} +} + +pub struct DummyMediaOutput; +impl MediaOutput for DummyMediaOutput { + fn add_stream(&mut self, _stream: &MediaStreamId) {} +} + +pub struct DummyWebRtcController; + +impl WebRtcControllerBackend for DummyWebRtcController { + fn configure(&mut self, _: &str, _: BundlePolicy) -> WebRtcResult { + Ok(()) + } + fn set_remote_description( + &mut self, + _: SessionDescription, + _: Box, + ) -> WebRtcResult { + Ok(()) + } + fn set_local_description( + &mut self, + _: SessionDescription, + _: Box, + ) -> WebRtcResult { + Ok(()) + } + fn add_ice_candidate(&mut self, _: IceCandidate) -> WebRtcResult { + Ok(()) + } + fn create_offer( + &mut self, + _: Box, + ) -> WebRtcResult { + Ok(()) + } + fn create_answer( + &mut self, + _: Box, + ) -> WebRtcResult { + Ok(()) + } + fn add_stream(&mut self, _: &MediaStreamId) -> WebRtcResult { + Ok(()) + } + fn create_data_channel(&mut self, _: &DataChannelInit) -> WebRtcDataChannelResult { + Ok(0) + } + fn close_data_channel(&mut self, _: &DataChannelId) -> WebRtcResult { + Ok(()) + } + fn send_data_channel_message( + &mut self, + _: &DataChannelId, + _: &DataChannelMessage, + ) -> WebRtcResult { + Ok(()) + } + fn internal_event(&mut self, _: thread::InternalEvent) -> WebRtcResult { + Ok(()) + } + fn quit(&mut self) {} +} + +impl MediaInstance for DummyPlayer { + fn get_id(&self) -> usize { + 0 + } + + fn mute(&self, _val: bool) -> Result<(), MediaInstanceError> { + Ok(()) + } + + fn suspend(&self) -> Result<(), MediaInstanceError> { + Ok(()) + } + + fn resume(&self) -> Result<(), MediaInstanceError> { + Ok(()) + } +} + +struct DummyMediaDeviceMonitor; + +impl MediaDeviceMonitor for DummyMediaDeviceMonitor { + fn enumerate_devices(&self) -> Option> { + Some(vec![]) + } +} diff --git a/components/media/backends/gstreamer/Cargo.toml b/components/media/backends/gstreamer/Cargo.toml new file mode 100644 index 00000000000..82f610a230f --- /dev/null +++ b/components/media/backends/gstreamer/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "servo-media-gstreamer" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_gstreamer" +path = "lib.rs" + +[features] +glx = ["servo-media-gstreamer-render-unix/gl-x11"] +wayland = ["servo-media-gstreamer-render-unix/gl-wayland"] + +[dependencies] +byte-slice-cast = "1" +glib = { workspace = true } +glib-sys = { workspace = true } +gstreamer = { workspace = true } +gstreamer-app = { workspace = true } +gstreamer-audio = { workspace = true } +gstreamer-video = { workspace = true } +gstreamer-base = { workspace = true } +gstreamer-play = { workspace = true } +gstreamer-webrtc = { workspace = true } +gstreamer-sdp = { workspace = true } +gstreamer-sys = { workspace = true } +ipc-channel = { workspace = true } +log = "0.4" +mime = "0.3.13" +once_cell = "1.18.0" +servo-media = { path = "../../servo-media" } +servo-media-audio = { path = "../../audio" } +servo-media-gstreamer-render = { path = "render" } +servo-media-player = { path = "../../player" } +servo-media-streams = { path = "../../streams" } +servo-media-traits = { path = "../../traits" } +servo-media-webrtc = { path = "../../webrtc" } +url = "2.0" + +[target.'cfg(any(target_os = "linux", target_os = "dragonfly", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))'.dependencies] +servo-media-gstreamer-render-unix = { path = "render-unix", features = [ + "gl-egl", +] } + +[target.'cfg(target_os = "android")'.dependencies] +servo-media-gstreamer-render-android = { path = "render-android" } diff --git a/components/media/backends/gstreamer/audio_decoder.rs b/components/media/backends/gstreamer/audio_decoder.rs new file mode 100644 index 00000000000..e9309765d55 --- /dev/null +++ b/components/media/backends/gstreamer/audio_decoder.rs @@ -0,0 +1,408 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::io::{Cursor, Read}; +use std::sync::{Arc, Mutex, mpsc}; + +use byte_slice_cast::*; +use gstreamer::prelude::*; +use servo_media_audio::decoder::{ + AudioDecoder, AudioDecoderCallbacks, AudioDecoderError, AudioDecoderOptions, +}; +use {gstreamer, gstreamer_app, gstreamer_audio}; + +pub struct GStreamerAudioDecoderProgress( + gstreamer::buffer::MappedBuffer, +); + +impl AsRef<[f32]> for GStreamerAudioDecoderProgress { + fn as_ref(&self) -> &[f32] { + self.0.as_ref().as_slice_of::().unwrap() + } +} + +#[derive(Default)] +pub struct GStreamerAudioDecoder {} + +impl GStreamerAudioDecoder { + pub fn new() -> Self { + Default::default() + } +} + +impl AudioDecoder for GStreamerAudioDecoder { + fn decode( + &self, + data: Vec, + callbacks: AudioDecoderCallbacks, + options: Option, + ) { + let pipeline = gstreamer::Pipeline::new(); + let callbacks = Arc::new(callbacks); + + let appsrc = match gstreamer::ElementFactory::make("appsrc").build() { + Ok(appsrc) => appsrc, + _ => { + return callbacks.error(AudioDecoderError::Backend( + "appsrc creation failed".to_owned(), + )); + }, + }; + + let decodebin = match gstreamer::ElementFactory::make("decodebin").build() { + Ok(decodebin) => decodebin, + _ => { + return callbacks.error(AudioDecoderError::Backend( + "decodebin creation failed".to_owned(), + )); + }, + }; + + // decodebin uses something called a "sometimes-pad", which is basically + // a pad that will show up when a certain condition is met, + // in decodebins case that is media being decoded + if let Err(e) = pipeline.add_many([&appsrc, &decodebin]) { + return callbacks.error(AudioDecoderError::Backend(e.to_string())); + } + + if let Err(e) = gstreamer::Element::link_many([&appsrc, &decodebin]) { + return callbacks.error(AudioDecoderError::Backend(e.to_string())); + } + + let appsrc = appsrc.downcast::().unwrap(); + + let options = options.unwrap_or_default(); + + let (sender, receiver) = mpsc::channel(); + let sender = Arc::new(Mutex::new(sender)); + + let pipeline_ = pipeline.downgrade(); + let callbacks_ = callbacks.clone(); + let sender_ = sender.clone(); + // Initial pipeline looks like + // + // appsrc ! decodebin2! ... + // + // We plug in the second part of the pipeline, including the deinterleave element, + // once the media starts being decoded. + decodebin.connect_pad_added(move |_, src_pad| { + // A decodebin pad was added, if this is an audio file, + // plug in a deinterleave element to separate each planar channel. + // + // Sub pipeline looks like + // + // ... decodebin2 ! audioconvert ! audioresample ! capsfilter ! deinterleave ... + // + // deinterleave also uses a sometime-pad, so we need to wait until + // a pad for a planar channel is added to plug in the last part of + // the pipeline, with the appsink that will be pulling the data from + // each channel. + + let callbacks = &callbacks_; + let sender = &sender_; + let pipeline = match pipeline_.upgrade() { + Some(pipeline) => pipeline, + None => { + callbacks.error(AudioDecoderError::Backend( + "Pipeline failed upgrade".to_owned(), + )); + let _ = sender.lock().unwrap().send(()); + return; + }, + }; + + let (is_audio, caps) = { + let media_type = src_pad.current_caps().and_then(|caps| { + caps.structure(0).map(|s| { + let name = s.name(); + (name.starts_with("audio/"), caps.clone()) + }) + }); + + match media_type { + None => { + callbacks.error(AudioDecoderError::Backend( + "Failed to get media type from pad".to_owned(), + )); + let _ = sender.lock().unwrap().send(()); + return; + }, + Some(media_type) => media_type, + } + }; + + if !is_audio { + callbacks.error(AudioDecoderError::InvalidMediaFormat); + let _ = sender.lock().unwrap().send(()); + return; + } + + let sample_audio_info = match gstreamer_audio::AudioInfo::from_caps(&caps) { + Ok(sample_audio_info) => sample_audio_info, + _ => { + callbacks.error(AudioDecoderError::Backend("AudioInfo failed".to_owned())); + let _ = sender.lock().unwrap().send(()); + return; + }, + }; + let channels = sample_audio_info.channels(); + callbacks.ready(channels); + + let insert_deinterleave = || -> Result<(), AudioDecoderError> { + let convert = gstreamer::ElementFactory::make("audioconvert") + .build() + .map_err(|error| { + AudioDecoderError::Backend(format!( + "audioconvert creation failed: {error:?}" + )) + })?; + let resample = gstreamer::ElementFactory::make("audioresample") + .build() + .map_err(|error| { + AudioDecoderError::Backend(format!( + "audioresample creation failed: {error:?}" + )) + })?; + let filter = gstreamer::ElementFactory::make("capsfilter") + .build() + .map_err(|error| { + AudioDecoderError::Backend(format!("capsfilter creation failed: {error:?}")) + })?; + let deinterleave = gstreamer::ElementFactory::make("deinterleave") + .name("deinterleave") + .property("keep-positions", true) + .build() + .map_err(|error| { + AudioDecoderError::Backend(format!( + "deinterleave creation failed: {error:?}" + )) + })?; + + let pipeline_ = pipeline.downgrade(); + let callbacks_ = callbacks.clone(); + deinterleave.connect_pad_added(move |_, src_pad| { + // A new pad for a planar channel was added in deinterleave. + // Plug in an appsink so we can pull the data from each channel. + // + // The end of the pipeline looks like: + // + // ... deinterleave ! queue ! appsink. + let callbacks = &callbacks_; + let pipeline = match pipeline_.upgrade() { + Some(pipeline) => pipeline, + None => { + return callbacks.error(AudioDecoderError::Backend( + "Pipeline failedupgrade".to_owned(), + )); + }, + }; + let insert_sink = || -> Result<(), AudioDecoderError> { + let queue = + gstreamer::ElementFactory::make("queue") + .build() + .map_err(|error| { + AudioDecoderError::Backend(format!( + "queue creation failed: {error:?}" + )) + })?; + let sink = gstreamer::ElementFactory::make("appsink").build().map_err( + |error| { + AudioDecoderError::Backend(format!( + "appsink creation failed: {error:?}" + )) + }, + )?; + let appsink = sink + .clone() + .dynamic_cast::() + .unwrap(); + sink.set_property("sync", false); + + let callbacks_ = callbacks.clone(); + appsink.set_callbacks( + gstreamer_app::AppSinkCallbacks::builder() + .new_sample(move |appsink| { + let sample = appsink + .pull_sample() + .map_err(|_| gstreamer::FlowError::Eos)?; + let buffer = sample.buffer_owned().ok_or_else(|| { + callbacks_.error(AudioDecoderError::InvalidSample); + gstreamer::FlowError::Error + })?; + + let audio_info = sample + .caps() + .and_then(|caps| { + gstreamer_audio::AudioInfo::from_caps(caps).ok() + }) + .ok_or_else(|| { + callbacks_.error(AudioDecoderError::Backend( + "Could not get caps from sample".to_owned(), + )); + gstreamer::FlowError::Error + })?; + let positions = audio_info.positions().ok_or_else(|| { + callbacks_.error(AudioDecoderError::Backend( + "AudioInfo failed".to_owned(), + )); + gstreamer::FlowError::Error + })?; + + for position in positions.iter() { + let buffer = buffer.clone(); + let map = match buffer.into_mapped_buffer_readable() { + Ok(map) => map, + _ => { + callbacks_ + .error(AudioDecoderError::BufferReadFailed); + return Err(gstreamer::FlowError::Error); + }, + }; + let progress = Box::new(GStreamerAudioDecoderProgress(map)); + let channel = position.to_mask() as u32; + callbacks_.progress(progress, channel); + } + + Ok(gstreamer::FlowSuccess::Ok) + }) + .build(), + ); + + let elements = &[&queue, &sink]; + pipeline + .add_many(elements) + .map_err(|e| AudioDecoderError::Backend(e.to_string()))?; + gstreamer::Element::link_many(elements) + .map_err(|e| AudioDecoderError::Backend(e.to_string()))?; + + for e in elements { + e.sync_state_with_parent() + .map_err(|e| AudioDecoderError::Backend(e.to_string()))?; + } + + let sink_pad = queue.static_pad("sink").ok_or( + AudioDecoderError::Backend("Could not get static pad sink".to_owned()), + )?; + src_pad.link(&sink_pad).map(|_| ()).map_err(|e| { + AudioDecoderError::Backend(format!("Sink pad link failed: {}", e)) + }) + }; + + if let Err(e) = insert_sink() { + callbacks.error(e); + } + }); + + let mut audio_info_builder = gstreamer_audio::AudioInfo::builder( + gstreamer_audio::AUDIO_FORMAT_F32, + options.sample_rate as u32, + channels, + ); + if let Some(positions) = sample_audio_info.positions() { + audio_info_builder = audio_info_builder.positions(positions); + } + let audio_info = audio_info_builder.build().map_err(|error| { + AudioDecoderError::Backend(format!("AudioInfo failed: {error:?}")) + })?; + let caps = audio_info.to_caps().map_err(|error| { + AudioDecoderError::Backend(format!("AudioInfo failed: {error:?}")) + })?; + filter.set_property("caps", caps); + + let elements = &[&convert, &resample, &filter, &deinterleave]; + pipeline + .add_many(elements) + .map_err(|e| AudioDecoderError::Backend(e.to_string()))?; + gstreamer::Element::link_many(elements) + .map_err(|e| AudioDecoderError::Backend(e.to_string()))?; + + for e in elements { + e.sync_state_with_parent() + .map_err(|e| AudioDecoderError::Backend(e.to_string()))?; + } + + let sink_pad = convert + .static_pad("sink") + .ok_or(AudioDecoderError::Backend( + "Get static pad sink failed".to_owned(), + ))?; + src_pad + .link(&sink_pad) + .map(|_| ()) + .map_err(|e| AudioDecoderError::Backend(format!("Sink pad link failed: {}", e))) + }; + + if let Err(e) = insert_deinterleave() { + callbacks.error(e); + let _ = sender.lock().unwrap().send(()); + } + }); + + appsrc.set_format(gstreamer::Format::Bytes); + appsrc.set_block(true); + + let bus = match pipeline.bus() { + Some(bus) => bus, + None => { + callbacks.error(AudioDecoderError::Backend( + "Pipeline without bus. Shouldn't happen!".to_owned(), + )); + let _ = sender.lock().unwrap().send(()); + return; + }, + }; + + let callbacks_ = callbacks.clone(); + bus.set_sync_handler(move |_, msg| { + use gstreamer::MessageView; + + match msg.view() { + MessageView::Error(e) => { + callbacks_.error(AudioDecoderError::Backend( + e.debug() + .map(|d| d.to_string()) + .unwrap_or_else(|| "Unknown".to_owned()), + )); + let _ = sender.lock().unwrap().send(()); + }, + MessageView::Eos(_) => { + callbacks_.eos(); + let _ = sender.lock().unwrap().send(()); + }, + _ => (), + } + gstreamer::BusSyncReply::Drop + }); + + if pipeline.set_state(gstreamer::State::Playing).is_err() { + callbacks.error(AudioDecoderError::StateChangeFailed); + return; + } + + let max_bytes = appsrc.max_bytes() as usize; + let data_len = data.len(); + let mut reader = Cursor::new(data); + while (reader.position() as usize) < data_len { + let data_left = data_len - reader.position() as usize; + let buffer_size = if data_left < max_bytes { + data_left + } else { + max_bytes + }; + let mut buffer = gstreamer::Buffer::with_size(buffer_size).unwrap(); + { + let buffer = buffer.get_mut().unwrap(); + let mut map = buffer.map_writable().unwrap(); + let buffer = map.as_mut_slice(); + let _ = reader.read(buffer); + } + let _ = appsrc.push_buffer(buffer); + } + let _ = appsrc.end_of_stream(); + + // Wait until we get an error or EOS. + receiver.recv().unwrap(); + let _ = pipeline.set_state(gstreamer::State::Null); + } +} diff --git a/components/media/backends/gstreamer/audio_sink.rs b/components/media/backends/gstreamer/audio_sink.rs new file mode 100644 index 00000000000..9bd9fd9205e --- /dev/null +++ b/components/media/backends/gstreamer/audio_sink.rs @@ -0,0 +1,252 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cell::{Cell, RefCell}; +use std::sync::Arc; +use std::sync::mpsc::Sender; +use std::thread::Builder; + +use byte_slice_cast::*; +use gstreamer::prelude::*; +use gstreamer_app::{AppSrc, AppSrcCallbacks}; +use servo_media_audio::block::{Chunk, FRAMES_PER_BLOCK}; +use servo_media_audio::render_thread::AudioRenderThreadMsg; +use servo_media_audio::sink::{AudioSink, AudioSinkError}; +use servo_media_streams::MediaSocket; +use {gstreamer, gstreamer_audio}; + +use crate::media_stream::GstreamerMediaSocket; + +const DEFAULT_SAMPLE_RATE: f32 = 44100.; + +pub struct GStreamerAudioSink { + pipeline: gstreamer::Pipeline, + appsrc: Arc, + sample_rate: Cell, + audio_info: RefCell>, + sample_offset: Cell, +} + +impl GStreamerAudioSink { + pub fn new() -> Result { + if let Some(category) = gstreamer::DebugCategory::get("openslessink") { + category.set_threshold(gstreamer::DebugLevel::Trace); + } + gstreamer::init().map_err(|error| { + AudioSinkError::Backend(format!("GStreamer init failed: {error:?}")) + })?; + + let appsrc = gstreamer::ElementFactory::make("appsrc") + .build() + .map_err(|error| { + AudioSinkError::Backend(format!("appsrc creation failed: {error:?}")) + })?; + let appsrc = appsrc.downcast::().unwrap(); + + Ok(Self { + pipeline: gstreamer::Pipeline::new(), + appsrc: Arc::new(appsrc), + sample_rate: Cell::new(DEFAULT_SAMPLE_RATE), + audio_info: RefCell::new(None), + sample_offset: Cell::new(0), + }) + } +} + +impl GStreamerAudioSink { + fn set_audio_info(&self, sample_rate: f32, channels: u8) -> Result<(), AudioSinkError> { + let audio_info = gstreamer_audio::AudioInfo::builder( + gstreamer_audio::AUDIO_FORMAT_F32, + sample_rate as u32, + channels.into(), + ) + .build() + .map_err(|error| AudioSinkError::Backend(format!("AudioInfo failed: {error:?}")))?; + self.appsrc.set_caps(audio_info.to_caps().ok().as_ref()); + *self.audio_info.borrow_mut() = Some(audio_info); + Ok(()) + } + + fn set_channels_if_changed(&self, channels: u8) -> Result<(), AudioSinkError> { + let curr_channels = match self.audio_info.borrow().as_ref() { + Some(ch) => ch.channels(), + _ => { + return Ok(()); + }, + }; + if channels != curr_channels as u8 { + self.set_audio_info(self.sample_rate.get(), channels)?; + } + Ok(()) + } +} + +impl AudioSink for GStreamerAudioSink { + fn init( + &self, + sample_rate: f32, + graph_thread_channel: Sender, + ) -> Result<(), AudioSinkError> { + self.sample_rate.set(sample_rate); + self.set_audio_info(sample_rate, 2)?; + self.appsrc.set_format(gstreamer::Format::Time); + + // Allow only a single chunk. + self.appsrc.set_max_bytes(1); + + let appsrc = self.appsrc.clone(); + Builder::new() + .name("GstAppSrcCallbacks".to_owned()) + .spawn(move || { + let need_data = move |_: &AppSrc, _: u32| { + if let Err(e) = graph_thread_channel.send(AudioRenderThreadMsg::SinkNeedData) { + log::warn!("Error sending need data event: {:?}", e); + } + }; + appsrc.set_callbacks(AppSrcCallbacks::builder().need_data(need_data).build()); + }) + .unwrap(); + + let appsrc = self.appsrc.as_ref().clone().upcast(); + let resample = gstreamer::ElementFactory::make("audioresample") + .build() + .map_err(|error| { + AudioSinkError::Backend(format!("audioresample creation failed: {error:?}")) + })?; + let convert = gstreamer::ElementFactory::make("audioconvert") + .build() + .map_err(|error| { + AudioSinkError::Backend(format!("audioconvert creation failed: {error:?}")) + })?; + let sink = gstreamer::ElementFactory::make("autoaudiosink") + .build() + .map_err(|error| { + AudioSinkError::Backend(format!("autoaudiosink creation failed: {error:?}")) + })?; + self.pipeline + .add_many([&appsrc, &resample, &convert, &sink]) + .map_err(|error| AudioSinkError::Backend(error.to_string()))?; + gstreamer::Element::link_many([&appsrc, &resample, &convert, &sink]) + .map_err(|error| AudioSinkError::Backend(error.to_string()))?; + + Ok(()) + } + + fn init_stream( + &self, + channels: u8, + sample_rate: f32, + socket: Box, + ) -> Result<(), AudioSinkError> { + self.sample_rate.set(sample_rate); + self.set_audio_info(sample_rate, channels)?; + self.appsrc.set_format(gstreamer::Format::Time); + + // Do not set max bytes or callback, we will push as needed + + let appsrc = self.appsrc.as_ref().clone().upcast(); + let convert = gstreamer::ElementFactory::make("audioconvert") + .build() + .map_err(|error| { + AudioSinkError::Backend(format!("audioconvert creation failed: {error:?}")) + })?; + let sink = socket + .as_any() + .downcast_ref::() + .unwrap() + .proxy_sink() + .clone(); + + self.pipeline + .add_many([&appsrc, &convert, &sink]) + .map_err(|error| AudioSinkError::Backend(error.to_string()))?; + gstreamer::Element::link_many([&appsrc, &convert, &sink]) + .map_err(|error| AudioSinkError::Backend(error.to_string()))?; + + Ok(()) + } + + fn play(&self) -> Result<(), AudioSinkError> { + self.pipeline + .set_state(gstreamer::State::Playing) + .map(|_| ()) + .map_err(|_| AudioSinkError::StateChangeFailed) + } + + fn stop(&self) -> Result<(), AudioSinkError> { + self.pipeline + .set_state(gstreamer::State::Paused) + .map(|_| ()) + .map_err(|_| AudioSinkError::StateChangeFailed) + } + + fn has_enough_data(&self) -> bool { + self.appsrc.current_level_bytes() >= self.appsrc.max_bytes() + } + + fn push_data(&self, mut chunk: Chunk) -> Result<(), AudioSinkError> { + if let Some(block) = chunk.blocks.first() { + self.set_channels_if_changed(block.chan_count())?; + } + + let sample_rate = self.sample_rate.get() as u64; + let audio_info = self.audio_info.borrow(); + let audio_info = audio_info.as_ref().unwrap(); + let channels = audio_info.channels(); + let bpf = audio_info.bpf() as usize; + assert_eq!(bpf, 4 * channels as usize); + let n_samples = FRAMES_PER_BLOCK.0; + let buf_size = (n_samples as usize) * (bpf); + let mut buffer = gstreamer::Buffer::with_size(buf_size).unwrap(); + { + let buffer = buffer.get_mut().unwrap(); + let mut sample_offset = self.sample_offset.get(); + // Calculate the current timestamp (PTS) and the next one, + // and calculate the duration from the difference instead of + // simply the number of samples to prevent rounding errors + let pts = gstreamer::ClockTime::from_nseconds( + sample_offset + .mul_div_floor(gstreamer::ClockTime::SECOND.nseconds(), sample_rate) + .unwrap(), + ); + let next_pts: gstreamer::ClockTime = gstreamer::ClockTime::from_nseconds( + (sample_offset + n_samples) + .mul_div_floor(gstreamer::ClockTime::SECOND.nseconds(), sample_rate) + .unwrap(), + ); + buffer.set_pts(Some(pts)); + buffer.set_duration(next_pts - pts); + + // sometimes nothing reaches the output + if chunk.is_empty() { + chunk.blocks.push(Default::default()); + chunk.blocks[0].repeat(channels as u8); + } + debug_assert!(chunk.len() == 1); + let mut data = chunk.blocks[0].interleave(); + let data = data.as_mut_byte_slice(); + + // XXXManishearth if we have a safe way to convert + // from Box<[f32]> to Box<[u8]> (similarly for Vec) + // we can use Buffer::from_slice instead + buffer.copy_from_slice(0, data).expect("copying failed"); + + sample_offset += n_samples; + self.sample_offset.set(sample_offset); + } + + self.appsrc + .push_buffer(buffer) + .map(|_| ()) + .map_err(|_| AudioSinkError::BufferPushFailed) + } + + fn set_eos_callback(&self, _: Box>) + Send + Sync + 'static>) {} +} + +impl Drop for GStreamerAudioSink { + fn drop(&mut self) { + let _ = self.stop(); + } +} diff --git a/components/media/backends/gstreamer/audio_stream_reader.rs b/components/media/backends/gstreamer/audio_stream_reader.rs new file mode 100644 index 00000000000..ae4b8820d29 --- /dev/null +++ b/components/media/backends/gstreamer/audio_stream_reader.rs @@ -0,0 +1,119 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::mpsc::{Receiver, channel}; + +use byte_slice_cast::*; +use gstreamer::Fraction; +use gstreamer::prelude::*; +use gstreamer_audio::AUDIO_FORMAT_F32; +use servo_media_audio::AudioStreamReader; +use servo_media_audio::block::{Block, FRAMES_PER_BLOCK_USIZE}; +use servo_media_streams::registry::{MediaStreamId, get_stream}; + +use crate::media_stream::GStreamerMediaStream; + +pub struct GStreamerAudioStreamReader { + rx: Receiver, + pipeline: gstreamer::Pipeline, +} + +impl GStreamerAudioStreamReader { + pub fn new(stream: MediaStreamId, sample_rate: f32) -> Result { + let (tx, rx) = channel(); + let stream = get_stream(&stream).unwrap(); + let mut stream = stream.lock().unwrap(); + let g_stream = stream + .as_mut_any() + .downcast_mut::() + .unwrap(); + let element = g_stream.src_element(); + let pipeline = g_stream.pipeline_or_new(); + drop(stream); + let time_per_block = Fraction::new(FRAMES_PER_BLOCK_USIZE as i32, sample_rate as i32); + + // XXXManishearth this is only necessary because of an upstream + // gstreamer bug. https://github.com/servo/media/pull/362#issuecomment-647947034 + let caps = gstreamer_audio::AudioCapsBuilder::new() + .layout(gstreamer_audio::AudioLayout::Interleaved) + .build(); + let capsfilter0 = gstreamer::ElementFactory::make("capsfilter") + .property("caps", caps) + .build() + .map_err(|error| format!("capsfilter creation failed: {error:?}"))?; + + let split = gstreamer::ElementFactory::make("audiobuffersplit") + .property("output-buffer-duration", time_per_block) + .build() + .map_err(|error| format!("audiobuffersplit creation failed: {error:?}"))?; + let convert = gstreamer::ElementFactory::make("audioconvert") + .build() + .map_err(|error| format!("audioconvert creation failed: {error:?}"))?; + let caps = gstreamer_audio::AudioCapsBuilder::new() + .layout(gstreamer_audio::AudioLayout::NonInterleaved) + .format(AUDIO_FORMAT_F32) + .rate(sample_rate as i32) + .build(); + let capsfilter = gstreamer::ElementFactory::make("capsfilter") + .property("caps", caps) + .build() + .map_err(|error| format!("capsfilter creation failed: {error:?}"))?; + let sink = gstreamer::ElementFactory::make("appsink") + .property("sync", false) + .build() + .map_err(|error| format!("appsink creation failed: {error:?}"))?; + + let appsink = sink + .clone() + .dynamic_cast::() + .unwrap(); + + let elements = [&element, &capsfilter0, &split, &convert, &capsfilter, &sink]; + pipeline + .add_many(&elements[1..]) + .map_err(|error| format!("pipeline adding failed: {error:?}"))?; + gstreamer::Element::link_many(elements) + .map_err(|error| format!("element linking failed: {error:?}"))?; + for e in &elements { + e.sync_state_with_parent().map_err(|e| e.to_string())?; + } + appsink.set_callbacks( + gstreamer_app::AppSinkCallbacks::builder() + .new_sample(move |appsink| { + let sample = appsink + .pull_sample() + .map_err(|_| gstreamer::FlowError::Eos)?; + let buffer = sample.buffer_owned().ok_or(gstreamer::FlowError::Error)?; + + let buffer = buffer + .into_mapped_buffer_readable() + .map_err(|_| gstreamer::FlowError::Error)?; + let floatref = buffer + .as_slice() + .as_slice_of::() + .map_err(|_| gstreamer::FlowError::Error)?; + + let block = Block::for_vec(floatref.into()); + tx.send(block).map_err(|_| gstreamer::FlowError::Error)?; + Ok(gstreamer::FlowSuccess::Ok) + }) + .build(), + ); + Ok(Self { rx, pipeline }) + } +} + +impl AudioStreamReader for GStreamerAudioStreamReader { + fn pull(&self) -> Block { + self.rx.recv().unwrap() + } + + fn start(&self) { + self.pipeline.set_state(gstreamer::State::Playing).unwrap(); + } + + fn stop(&self) { + self.pipeline.set_state(gstreamer::State::Null).unwrap(); + } +} diff --git a/components/media/backends/gstreamer/datachannel.rs b/components/media/backends/gstreamer/datachannel.rs new file mode 100644 index 00000000000..4c02a107405 --- /dev/null +++ b/components/media/backends/gstreamer/datachannel.rs @@ -0,0 +1,177 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::Mutex; + +use glib::prelude::*; +use gstreamer_webrtc::{WebRTCDataChannel, WebRTCDataChannelState}; +use servo_media_webrtc::thread::InternalEvent; +use servo_media_webrtc::{ + DataChannelEvent, DataChannelId, DataChannelInit, DataChannelMessage, DataChannelState, + WebRtcController as WebRtcThread, WebRtcError, +}; + +pub struct GStreamerWebRtcDataChannel { + channel: WebRTCDataChannel, + id: DataChannelId, + thread: WebRtcThread, +} + +impl GStreamerWebRtcDataChannel { + pub fn new( + servo_channel_id: &DataChannelId, + webrtc: &gstreamer::Element, + thread: &WebRtcThread, + init: &DataChannelInit, + ) -> Result { + let label = &init.label; + let mut init_struct = gstreamer::Structure::builder("options") + .field("ordered", init.ordered) + .field("protocol", &init.protocol) + .field("negotiated", init.negotiated) + .build(); + + if let Some(max_packet_life_time) = init.max_packet_life_time { + init_struct.set_value( + "max-packet-lifetime", + (max_packet_life_time as u32).to_send_value(), + ); + } + + if let Some(max_retransmits) = init.max_retransmits { + init_struct.set_value("max-retransmits", (max_retransmits as u32).to_send_value()); + } + + if let Some(id) = init.id { + init_struct.set_value("id", (id as u32).to_send_value()); + } + + let channel = webrtc + .emit_by_name::("create-data-channel", &[&label, &init_struct]); + + GStreamerWebRtcDataChannel::from(servo_channel_id, channel, thread) + } + + pub fn from( + id: &DataChannelId, + channel: WebRTCDataChannel, + thread: &WebRtcThread, + ) -> Result { + let id_ = *id; + let thread_ = Mutex::new(thread.clone()); + channel.connect_on_open(move |_| { + thread_ + .lock() + .unwrap() + .internal_event(InternalEvent::OnDataChannelEvent( + id_, + DataChannelEvent::Open, + )); + }); + + let id_ = *id; + let thread_ = Mutex::new(thread.clone()); + channel.connect_on_close(move |_| { + thread_ + .lock() + .unwrap() + .internal_event(InternalEvent::OnDataChannelEvent( + id_, + DataChannelEvent::Close, + )); + }); + + let id_ = *id; + let thread_ = Mutex::new(thread.clone()); + channel.connect_on_error(move |_, error| { + thread_ + .lock() + .unwrap() + .internal_event(InternalEvent::OnDataChannelEvent( + id_, + DataChannelEvent::Error(WebRtcError::Backend(error.to_string())), + )); + }); + + let id_ = *id; + let thread_ = Mutex::new(thread.clone()); + channel.connect_on_message_string(move |_, message| { + let Some(message) = message.map(|s| s.to_owned()) else { + return; + }; + thread_ + .lock() + .unwrap() + .internal_event(InternalEvent::OnDataChannelEvent( + id_, + DataChannelEvent::OnMessage(DataChannelMessage::Text(message)), + )); + }); + + let id_ = *id; + let thread_ = Mutex::new(thread.clone()); + channel.connect_on_message_data(move |_, message| { + let Some(message) = message.map(|b| b.to_owned()) else { + return; + }; + thread_ + .lock() + .unwrap() + .internal_event(InternalEvent::OnDataChannelEvent( + id_, + DataChannelEvent::OnMessage(DataChannelMessage::Binary(message.to_vec())), + )); + }); + + let id_ = *id; + let thread_ = Mutex::new(thread.clone()); + channel.connect_ready_state_notify(move |channel| { + let ready_state = channel.ready_state(); + let ready_state = match ready_state { + WebRTCDataChannelState::Connecting => DataChannelState::Connecting, + WebRTCDataChannelState::Open => DataChannelState::Open, + WebRTCDataChannelState::Closing => DataChannelState::Closing, + WebRTCDataChannelState::Closed => DataChannelState::Closed, + WebRTCDataChannelState::__Unknown(state) => DataChannelState::__Unknown(state), + _ => return, + }; + thread_ + .lock() + .unwrap() + .internal_event(InternalEvent::OnDataChannelEvent( + id_, + DataChannelEvent::StateChange(ready_state), + )); + }); + + Ok(Self { + id: *id, + thread: thread.to_owned(), + channel, + }) + } + + pub fn send(&self, message: &DataChannelMessage) { + match message { + DataChannelMessage::Text(text) => self.channel.send_string(Some(text)), + DataChannelMessage::Binary(data) => self + .channel + .send_data(Some(&glib::Bytes::from(data.as_slice()))), + } + } + + pub fn close(&self) { + self.channel.close() + } +} + +impl Drop for GStreamerWebRtcDataChannel { + fn drop(&mut self) { + self.thread + .internal_event(InternalEvent::OnDataChannelEvent( + self.id, + DataChannelEvent::Close, + )); + } +} diff --git a/components/media/backends/gstreamer/device_monitor.rs b/components/media/backends/gstreamer/device_monitor.rs new file mode 100644 index 00000000000..dc4fd712e34 --- /dev/null +++ b/components/media/backends/gstreamer/device_monitor.rs @@ -0,0 +1,64 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cell::RefCell; + +use gstreamer::DeviceMonitor as GstDeviceMonitor; +use gstreamer::prelude::*; +use servo_media_streams::device_monitor::{MediaDeviceInfo, MediaDeviceKind, MediaDeviceMonitor}; + +pub struct GStreamerDeviceMonitor { + devices: RefCell>>, +} + +impl GStreamerDeviceMonitor { + pub fn new() -> Self { + Self { + devices: RefCell::new(None), + } + } + + fn get_devices(&self) -> Result, ()> { + const AUDIO_SOURCE: &str = "Audio/Source"; + const AUDIO_SINK: &str = "Audio/Sink"; + const VIDEO_SOURCE: &str = "Video/Source"; + let device_monitor = GstDeviceMonitor::new(); + let audio_caps = gstreamer_audio::AudioCapsBuilder::new().build(); + device_monitor.add_filter(Some(AUDIO_SOURCE), Some(&audio_caps)); + device_monitor.add_filter(Some(AUDIO_SINK), Some(&audio_caps)); + let video_caps = gstreamer_video::VideoCapsBuilder::new().build(); + device_monitor.add_filter(Some(VIDEO_SOURCE), Some(&video_caps)); + let devices = device_monitor + .devices() + .iter() + .filter_map(|device| { + let display_name = device.display_name().as_str().to_owned(); + Some(MediaDeviceInfo { + device_id: display_name.clone(), + kind: match device.device_class().as_str() { + AUDIO_SOURCE => MediaDeviceKind::AudioInput, + AUDIO_SINK => MediaDeviceKind::AudioOutput, + VIDEO_SOURCE => MediaDeviceKind::VideoInput, + _ => return None, + }, + label: display_name, + }) + }) + .collect(); + Ok(devices) + } +} + +impl MediaDeviceMonitor for GStreamerDeviceMonitor { + fn enumerate_devices(&self) -> Option> { + { + if let Some(ref devices) = *self.devices.borrow() { + return Some(devices.clone()); + } + } + let devices = self.get_devices().ok()?; + *self.devices.borrow_mut() = Some(devices.clone()); + Some(devices) + } +} diff --git a/components/media/backends/gstreamer/lib.rs b/components/media/backends/gstreamer/lib.rs new file mode 100644 index 00000000000..f01cd4a34e0 --- /dev/null +++ b/components/media/backends/gstreamer/lib.rs @@ -0,0 +1,359 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +pub mod audio_decoder; +pub mod audio_sink; +pub mod audio_stream_reader; +mod datachannel; +mod device_monitor; +pub mod media_capture; +pub mod media_stream; +mod media_stream_source; +pub mod player; +mod registry_scanner; +mod render; +mod source; +pub mod webrtc; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::{self, Sender}; +use std::sync::{Arc, Mutex, Weak}; +use std::thread; +use std::vec::Vec; + +use device_monitor::GStreamerDeviceMonitor; +use gstreamer::prelude::*; +use ipc_channel::ipc::IpcSender; +use log::warn; +use media_stream::GStreamerMediaStream; +use mime::Mime; +use once_cell::sync::{Lazy, OnceCell}; +use registry_scanner::GSTREAMER_REGISTRY_SCANNER; +use servo_media::{Backend, BackendDeInit, BackendInit, MediaInstanceError, SupportsMediaType}; +use servo_media_audio::context::{AudioContext, AudioContextOptions}; +use servo_media_audio::decoder::AudioDecoder; +use servo_media_audio::sink::AudioSinkError; +use servo_media_audio::{AudioBackend, AudioStreamReader}; +use servo_media_player::audio::AudioRenderer; +use servo_media_player::context::PlayerGLContext; +use servo_media_player::video::VideoFrameRenderer; +use servo_media_player::{Player, PlayerEvent, StreamType}; +use servo_media_streams::capture::MediaTrackConstraintSet; +use servo_media_streams::device_monitor::MediaDeviceMonitor; +use servo_media_streams::registry::MediaStreamId; +use servo_media_streams::{MediaOutput, MediaSocket, MediaStreamType}; +use servo_media_traits::{BackendMsg, ClientContextId, MediaInstance}; +use servo_media_webrtc::{WebRtcBackend, WebRtcController, WebRtcSignaller}; + +static BACKEND_BASE_TIME: Lazy = + Lazy::new(|| gstreamer::SystemClock::obtain().time()); + +static BACKEND_THREAD: OnceCell = OnceCell::new(); + +pub type WeakMediaInstance = Weak>; +pub type WeakMediaInstanceHashMap = HashMap>; + +pub struct GStreamerBackend { + capture_mocking: AtomicBool, + instances: Arc>, + next_instance_id: AtomicUsize, + /// Channel to communicate media instances with its owner Backend. + backend_chan: Arc>>, +} + +#[derive(Debug)] +#[allow(dead_code)] +pub struct ErrorLoadingPlugins(Vec<&'static str>); + +impl GStreamerBackend { + pub fn init_with_plugins( + plugin_dir: PathBuf, + plugins: &[&'static str], + ) -> Result, ErrorLoadingPlugins> { + gstreamer::init().unwrap(); + + // GStreamer between 1.19.1 and 1.22.7 will not send messages like "end of stream" + // to GstPlayer unless there is a GLib main loop running somewhere. We should remove + // this workaround when we raise of required version of GStreamer. + // See https://github.com/servo/media/pull/393. + let needs_background_glib_main_loop = { + let (major, minor, micro, _) = gstreamer::version(); + (major, minor, micro) >= (1, 19, 1) && (major, minor, micro) <= (1, 22, 7) + }; + + if needs_background_glib_main_loop { + BACKEND_THREAD.get_or_init(|| { + thread::spawn(|| glib::MainLoop::new(None, false).run()); + true + }); + } + + let mut errors = vec![]; + for plugin in plugins { + let mut path = plugin_dir.clone(); + path.push(plugin); + let registry = gstreamer::Registry::get(); + if gstreamer::Plugin::load_file(&path) + .is_ok_and(|plugin| registry.add_plugin(&plugin).is_ok()) + { + continue; + } + errors.push(*plugin); + } + + if !errors.is_empty() { + return Err(ErrorLoadingPlugins(errors)); + } + + type MediaInstancesVec = Vec<(usize, Weak>)>; + let instances: HashMap = Default::default(); + let instances = Arc::new(Mutex::new(instances)); + + let instances_ = instances.clone(); + let (backend_chan, recvr) = mpsc::channel(); + thread::Builder::new() + .name("GStreamerBackend ShutdownThread".to_owned()) + .spawn(move || { + match recvr.recv().unwrap() { + BackendMsg::Shutdown { + context, + id, + tx_ack, + } => { + let mut instances_ = instances_.lock().unwrap(); + if let Some(vec) = instances_.get_mut(&context) { + vec.retain(|m| m.0 != id); + if vec.is_empty() { + instances_.remove(&context); + } + } + // tell caller we are done removing this instance + let _ = tx_ack.send(()); + }, + }; + }) + .unwrap(); + + Ok(Box::new(GStreamerBackend { + capture_mocking: AtomicBool::new(false), + instances, + next_instance_id: AtomicUsize::new(0), + backend_chan: Arc::new(Mutex::new(backend_chan)), + })) + } + + fn media_instance_action( + &self, + id: &ClientContextId, + cb: &dyn Fn(&dyn MediaInstance) -> Result<(), MediaInstanceError>, + ) { + let mut instances = self.instances.lock().unwrap(); + match instances.get_mut(id) { + Some(vec) => vec.retain(|(_, weak)| match weak.upgrade() { + Some(instance) => { + if cb(&*(instance.lock().unwrap())).is_err() { + warn!("Error executing media instance action"); + } + true + }, + _ => false, + }), + None => { + warn!("Trying to exec media action on an unknown client context"); + }, + } + } +} + +impl Backend for GStreamerBackend { + fn create_player( + &self, + context_id: &ClientContextId, + stream_type: StreamType, + sender: IpcSender, + renderer: Option>>, + audio_renderer: Option>>, + gl_context: Box, + ) -> Arc> { + let id = self.next_instance_id.fetch_add(1, Ordering::Relaxed); + let player = Arc::new(Mutex::new(player::GStreamerPlayer::new( + id, + context_id, + self.backend_chan.clone(), + stream_type, + sender, + renderer, + audio_renderer, + gl_context, + ))); + let mut instances = self.instances.lock().unwrap(); + let entry = instances.entry(*context_id).or_default(); + entry.push((id, Arc::downgrade(&player).clone())); + player + } + + fn create_audio_context( + &self, + client_context_id: &ClientContextId, + options: AudioContextOptions, + ) -> Result>, AudioSinkError> { + let id = self.next_instance_id.fetch_add(1, Ordering::Relaxed); + let audio_context = + AudioContext::new::(id, client_context_id, self.backend_chan.clone(), options)?; + + let audio_context = Arc::new(Mutex::new(audio_context)); + + let mut instances = self.instances.lock().unwrap(); + let entry = instances.entry(*client_context_id).or_default(); + entry.push((id, Arc::downgrade(&audio_context).clone())); + + Ok(audio_context) + } + + fn create_webrtc(&self, signaller: Box) -> WebRtcController { + WebRtcController::new::(signaller) + } + + fn create_audiostream(&self) -> MediaStreamId { + GStreamerMediaStream::create_audio() + } + + fn create_videostream(&self) -> MediaStreamId { + GStreamerMediaStream::create_video() + } + + fn create_stream_output(&self) -> Box { + Box::new(media_stream::MediaSink::default()) + } + + fn create_stream_and_socket( + &self, + ty: MediaStreamType, + ) -> (Box, MediaStreamId) { + let (id, socket) = GStreamerMediaStream::create_proxy(ty); + (Box::new(socket), id) + } + + fn create_audioinput_stream(&self, set: MediaTrackConstraintSet) -> Option { + if self.capture_mocking.load(Ordering::Acquire) { + // XXXManishearth we should caps filter this + return Some(self.create_audiostream()); + } + media_capture::create_audioinput_stream(set) + } + + fn create_videoinput_stream(&self, set: MediaTrackConstraintSet) -> Option { + if self.capture_mocking.load(Ordering::Acquire) { + // XXXManishearth we should caps filter this + return Some(self.create_videostream()); + } + media_capture::create_videoinput_stream(set) + } + + fn can_play_type(&self, media_type: &str) -> SupportsMediaType { + if let Ok(mime) = media_type.parse::() { + let mime_type = mime.type_().as_str().to_owned() + "/" + mime.subtype().as_str(); + let codecs = match mime.get_param("codecs") { + Some(codecs) => codecs + .as_str() + .split(',') + .map(|codec| codec.trim()) + .collect(), + None => vec![], + }; + + if GSTREAMER_REGISTRY_SCANNER.is_container_type_supported(&mime_type) { + if codecs.is_empty() { + return SupportsMediaType::Maybe; + } else if GSTREAMER_REGISTRY_SCANNER.are_all_codecs_supported(&codecs) { + return SupportsMediaType::Probably; + } else { + return SupportsMediaType::No; + } + } + } + SupportsMediaType::No + } + + fn set_capture_mocking(&self, mock: bool) { + self.capture_mocking.store(mock, Ordering::Release) + } + + fn mute(&self, id: &ClientContextId, val: bool) { + self.media_instance_action( + id, + &(move |instance: &dyn MediaInstance| instance.mute(val)), + ); + } + + fn suspend(&self, id: &ClientContextId) { + self.media_instance_action(id, &|instance: &dyn MediaInstance| instance.suspend()); + } + + fn resume(&self, id: &ClientContextId) { + self.media_instance_action(id, &|instance: &dyn MediaInstance| instance.resume()); + } + + fn get_device_monitor(&self) -> Box { + Box::new(GStreamerDeviceMonitor::new()) + } +} + +impl AudioBackend for GStreamerBackend { + type Sink = audio_sink::GStreamerAudioSink; + fn make_decoder() -> Box { + Box::new(audio_decoder::GStreamerAudioDecoder::new()) + } + fn make_sink() -> Result { + audio_sink::GStreamerAudioSink::new() + } + + fn make_streamreader(id: MediaStreamId, sample_rate: f32) -> Box { + Box::new(audio_stream_reader::GStreamerAudioStreamReader::new(id, sample_rate).unwrap()) + } +} + +impl WebRtcBackend for GStreamerBackend { + type Controller = webrtc::GStreamerWebRtcController; + + fn construct_webrtc_controller( + signaller: Box, + thread: WebRtcController, + ) -> Self::Controller { + webrtc::construct(signaller, thread).expect("WebRTC creation failed") + } +} + +impl BackendInit for GStreamerBackend { + fn init() -> Box { + Self::init_with_plugins(PathBuf::new(), &[]).unwrap() + } +} + +impl BackendDeInit for GStreamerBackend { + fn deinit(&self) { + let to_shutdown: Vec<(ClientContextId, usize)> = { + let map = self.instances.lock().unwrap(); + map.iter() + .flat_map(|(ctx, v)| v.iter().map(move |(id, _)| (*ctx, *id))) + .collect() + }; + + for (ctx, id) in to_shutdown { + let (tx_ack, rx_ack) = mpsc::channel(); + let _ = self + .backend_chan + .lock() + .unwrap() + .send(BackendMsg::Shutdown { + context: ctx, + id, + tx_ack, + }); + let _ = rx_ack.recv(); + } + } +} diff --git a/components/media/backends/gstreamer/media_capture.rs b/components/media/backends/gstreamer/media_capture.rs new file mode 100644 index 00000000000..0e8f1870e01 --- /dev/null +++ b/components/media/backends/gstreamer/media_capture.rs @@ -0,0 +1,172 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use gstreamer; +use gstreamer::caps::NoFeature; +use gstreamer::prelude::*; +use servo_media_streams::MediaStreamType; +use servo_media_streams::capture::*; +use servo_media_streams::registry::MediaStreamId; + +use crate::media_stream::GStreamerMediaStream; + +trait AddToCaps { + type Bound; + fn add_to_caps( + &self, + name: &str, + min: Self::Bound, + max: Self::Bound, + builder: gstreamer::caps::Builder, + ) -> Option>; +} + +impl AddToCaps for Constrain { + type Bound = u32; + fn add_to_caps( + &self, + name: &str, + min: u32, + max: u32, + builder: gstreamer::caps::Builder, + ) -> Option> { + match self { + Constrain::Value(v) => Some(builder.field(name, v)), + Constrain::Range(r) => { + let min = into_i32(r.min.unwrap_or(min)); + let max = into_i32(r.max.unwrap_or(max)); + let range = gstreamer::IntRange::::new(min, max); + + // TODO: Include the ideal caps value in the caps, needs a refactor + // of the AddToCaps trait + Some(builder.field(name, range)) + }, + } + } +} + +fn into_i32(x: u32) -> i32 { + if x > i32::MAX as u32 { + i32::MAX + } else { + x as i32 + } +} + +impl AddToCaps for Constrain { + type Bound = i32; + fn add_to_caps<'a>( + &self, + name: &str, + min: i32, + max: i32, + builder: gstreamer::caps::Builder, + ) -> Option> { + match self { + Constrain::Value(v) => { + Some(builder.field("name", gstreamer::Fraction::approximate_f64(*v)?)) + }, + Constrain::Range(r) => { + let min = r + .min + .and_then(gstreamer::Fraction::approximate_f64) + .unwrap_or(gstreamer::Fraction::new(min, 1)); + let max = r + .max + .and_then(gstreamer::Fraction::approximate_f64) + .unwrap_or(gstreamer::Fraction::new(max, 1)); + let range = gstreamer::FractionRange::new(min, max); + // TODO: Include the ideal caps value in the caps, needs a refactor + // of the AddToCaps trait + Some(builder.field(name, range)) + }, + } + } +} + +// TODO(Manishearth): Should support a set of constraints +fn into_caps(set: MediaTrackConstraintSet, format: &str) -> Option { + let mut builder = gstreamer::Caps::builder(format); + if let Some(w) = set.width { + builder = w.add_to_caps("width", 0, 1000000, builder)?; + } + if let Some(h) = set.height { + builder = h.add_to_caps("height", 0, 1000000, builder)?; + } + if let Some(aspect) = set.aspect { + builder = aspect.add_to_caps("pixel-aspect-ratio", 0, 1000000, builder)?; + } + if let Some(fr) = set.frame_rate { + builder = fr.add_to_caps("framerate", 0, 1000000, builder)?; + } + if let Some(sr) = set.sample_rate { + builder = sr.add_to_caps("rate", 0, 1000000, builder)?; + } + Some(builder.build()) +} + +struct GstMediaDevices { + monitor: gstreamer::DeviceMonitor, +} + +impl GstMediaDevices { + pub fn new() -> Self { + Self { + monitor: gstreamer::DeviceMonitor::new(), + } + } + + pub fn get_track( + &self, + video: bool, + constraints: MediaTrackConstraintSet, + ) -> Option { + let (format, filter) = if video { + ("video/x-raw", "Video/Source") + } else { + ("audio/x-raw", "Audio/Source") + }; + let caps = into_caps(constraints, format)?; + let f = self.monitor.add_filter(Some(filter), Some(&caps)); + let devices = self.monitor.devices(); + if let Some(f) = f { + let _ = self.monitor.remove_filter(f); + } + match devices.front() { + Some(d) => { + let element = d.create_element(None).ok()?; + Some(GstMediaTrack { element }) + }, + _ => None, + } + } +} + +pub struct GstMediaTrack { + element: gstreamer::Element, +} + +fn create_input_stream( + stream_type: MediaStreamType, + constraint_set: MediaTrackConstraintSet, +) -> Option { + let devices = GstMediaDevices::new(); + devices + .get_track(stream_type == MediaStreamType::Video, constraint_set) + .map(|track| { + let f = match stream_type { + MediaStreamType::Audio => GStreamerMediaStream::create_audio_from, + MediaStreamType::Video => GStreamerMediaStream::create_video_from, + }; + f(track.element) + }) +} + +pub fn create_audioinput_stream(constraint_set: MediaTrackConstraintSet) -> Option { + create_input_stream(MediaStreamType::Audio, constraint_set) +} + +pub fn create_videoinput_stream(constraint_set: MediaTrackConstraintSet) -> Option { + create_input_stream(MediaStreamType::Video, constraint_set) +} diff --git a/components/media/backends/gstreamer/media_stream.rs b/components/media/backends/gstreamer/media_stream.rs new file mode 100644 index 00000000000..29a4ed45267 --- /dev/null +++ b/components/media/backends/gstreamer/media_stream.rs @@ -0,0 +1,298 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::any::Any; +use std::sync::{Arc, Mutex}; + +use gstreamer; +use gstreamer::prelude::*; +use once_cell::sync::Lazy; +use servo_media_streams::registry::{ + MediaStreamId, get_stream, register_stream, unregister_stream, +}; +use servo_media_streams::{MediaOutput, MediaSocket, MediaStream, MediaStreamType}; + +use super::BACKEND_BASE_TIME; + +pub static RTP_CAPS_OPUS: Lazy = Lazy::new(|| { + gstreamer::Caps::builder("application/x-rtp") + .field("media", "audio") + .field("encoding-name", "OPUS") + .build() +}); + +pub static RTP_CAPS_VP8: Lazy = Lazy::new(|| { + gstreamer::Caps::builder("application/x-rtp") + .field("media", "video") + .field("encoding-name", "VP8") + .build() +}); + +pub struct GStreamerMediaStream { + id: Option, + type_: MediaStreamType, + elements: Vec, + pipeline: Option, +} + +impl MediaStream for GStreamerMediaStream { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_mut_any(&mut self) -> &mut dyn Any { + self + } + + fn set_id(&mut self, id: MediaStreamId) { + self.id = Some(id); + } + + fn ty(&self) -> MediaStreamType { + self.type_ + } +} + +impl GStreamerMediaStream { + pub fn new(type_: MediaStreamType, elements: Vec) -> Self { + Self { + id: None, + type_, + elements, + pipeline: None, + } + } + + pub fn caps(&self) -> &gstreamer::Caps { + match self.type_ { + MediaStreamType::Audio => &RTP_CAPS_OPUS, + MediaStreamType::Video => &RTP_CAPS_VP8, + } + } + + pub fn caps_with_payload(&self, payload: i32) -> gstreamer::Caps { + match self.type_ { + MediaStreamType::Audio => gstreamer::Caps::builder("application/x-rtp") + .field("media", "audio") + .field("encoding-name", "OPUS") + .field("payload", payload) + .build(), + MediaStreamType::Video => gstreamer::Caps::builder("application/x-rtp") + .field("media", "video") + .field("encoding-name", "VP8") + .field("payload", payload) + .build(), + } + } + + pub fn src_element(&self) -> gstreamer::Element { + self.elements.last().unwrap().clone() + } + + pub fn attach_to_pipeline(&mut self, pipeline: &gstreamer::Pipeline) { + assert!(self.pipeline.is_none()); + let elements: Vec<_> = self.elements.iter().collect(); + pipeline.add_many(&elements[..]).unwrap(); + gstreamer::Element::link_many(&elements[..]).unwrap(); + for element in elements { + element.sync_state_with_parent().unwrap(); + } + self.pipeline = Some(pipeline.clone()); + } + + pub fn pipeline_or_new(&mut self) -> gstreamer::Pipeline { + match self.pipeline { + Some(ref pipeline) => pipeline.clone(), + _ => { + let pipeline = + gstreamer::Pipeline::with_name("gstreamermediastream fresh pipeline"); + let clock = gstreamer::SystemClock::obtain(); + pipeline.set_start_time(gstreamer::ClockTime::NONE); + pipeline.set_base_time(*BACKEND_BASE_TIME); + pipeline.use_clock(Some(&clock)); + self.attach_to_pipeline(&pipeline); + pipeline + }, + } + } + + pub fn create_video() -> MediaStreamId { + let videotestsrc = gstreamer::ElementFactory::make("videotestsrc") + .property_from_str("pattern", "ball") + .property("is-live", true) + .build() + .unwrap(); + Self::create_video_from(videotestsrc) + } + + /// Attaches encoding adapters to the stream, returning the source element + pub fn encoded(&mut self) -> gstreamer::Element { + let pipeline = self + .pipeline + .as_ref() + .expect("GStreamerMediaStream::encoded() should not be called without a pipeline"); + let src = self.src_element(); + + let capsfilter = gstreamer::ElementFactory::make("capsfilter") + .property("caps", self.caps()) + .build() + .unwrap(); + match self.type_ { + MediaStreamType::Video => { + let vp8enc = gstreamer::ElementFactory::make("vp8enc") + .property("deadline", 1i64) + .property("error-resilient", "default") + .property("cpu-used", -16i32) + .property("lag-in-frames", 0i32) + .build() + .unwrap(); + + let rtpvp8pay = gstreamer::ElementFactory::make("rtpvp8pay") + .property("picture-id-mode", "15-bit") + .property("mtu", 1200u32) + .build() + .unwrap(); + let queue2 = gstreamer::ElementFactory::make("queue").build().unwrap(); + + pipeline + .add_many([&vp8enc, &rtpvp8pay, &queue2, &capsfilter]) + .unwrap(); + gstreamer::Element::link_many([&src, &vp8enc, &rtpvp8pay, &queue2, &capsfilter]) + .unwrap(); + vp8enc.sync_state_with_parent().unwrap(); + rtpvp8pay.sync_state_with_parent().unwrap(); + queue2.sync_state_with_parent().unwrap(); + capsfilter.sync_state_with_parent().unwrap(); + capsfilter + }, + MediaStreamType::Audio => { + let opusenc = gstreamer::ElementFactory::make("opusenc").build().unwrap(); + let rtpopuspay = gstreamer::ElementFactory::make("rtpopuspay") + .property("mtu", 1200u32) + .build() + .unwrap(); + let queue3 = gstreamer::ElementFactory::make("queue").build().unwrap(); + pipeline + .add_many([&opusenc, &rtpopuspay, &queue3, &capsfilter]) + .unwrap(); + gstreamer::Element::link_many([&src, &opusenc, &rtpopuspay, &queue3, &capsfilter]) + .unwrap(); + opusenc.sync_state_with_parent().unwrap(); + rtpopuspay.sync_state_with_parent().unwrap(); + queue3.sync_state_with_parent().unwrap(); + capsfilter + }, + } + } + + pub fn create_video_from(source: gstreamer::Element) -> MediaStreamId { + let videoconvert = gstreamer::ElementFactory::make("videoconvert") + .build() + .unwrap(); + let queue = gstreamer::ElementFactory::make("queue").build().unwrap(); + + register_stream(Arc::new(Mutex::new(GStreamerMediaStream::new( + MediaStreamType::Video, + vec![source, videoconvert, queue], + )))) + } + + pub fn create_audio() -> MediaStreamId { + let audiotestsrc = gstreamer::ElementFactory::make("audiotestsrc") + .property_from_str("wave", "sine") + .property("is-live", true) + .build() + .unwrap(); + + Self::create_audio_from(audiotestsrc) + } + + pub fn create_audio_from(source: gstreamer::Element) -> MediaStreamId { + let queue = gstreamer::ElementFactory::make("queue").build().unwrap(); + let audioconvert = gstreamer::ElementFactory::make("audioconvert") + .build() + .unwrap(); + let audioresample = gstreamer::ElementFactory::make("audioresample") + .build() + .unwrap(); + let queue2 = gstreamer::ElementFactory::make("queue").build().unwrap(); + + register_stream(Arc::new(Mutex::new(GStreamerMediaStream::new( + MediaStreamType::Audio, + vec![source, queue, audioconvert, audioresample, queue2], + )))) + } + + pub fn create_proxy(ty: MediaStreamType) -> (MediaStreamId, GstreamerMediaSocket) { + let proxy_sink = gstreamer::ElementFactory::make("proxysink") + .build() + .unwrap(); + let proxy_src = gstreamer::ElementFactory::make("proxysrc") + .property("proxysink", &proxy_sink) + .build() + .unwrap(); + let stream = match ty { + MediaStreamType::Audio => Self::create_audio_from(proxy_src), + MediaStreamType::Video => Self::create_video_from(proxy_src), + }; + + (stream, GstreamerMediaSocket { proxy_sink }) + } +} + +impl Drop for GStreamerMediaStream { + fn drop(&mut self) { + if let Some(ref id) = self.id { + unregister_stream(id); + } + } +} + +#[derive(Default)] +pub struct MediaSink { + streams: Vec>>, +} + +impl MediaOutput for MediaSink { + fn add_stream(&mut self, stream: &MediaStreamId) { + let stream = get_stream(stream).expect("Media streams registry does not contain such ID"); + { + let mut stream = stream.lock().unwrap(); + let stream = stream + .as_mut_any() + .downcast_mut::() + .unwrap(); + let pipeline = stream.pipeline_or_new(); + let last_element = stream.elements.last(); + let last_element = last_element.as_ref().unwrap(); + let sink = match stream.type_ { + MediaStreamType::Audio => "autoaudiosink", + MediaStreamType::Video => "autovideosink", + }; + let sink = gstreamer::ElementFactory::make(sink).build().unwrap(); + pipeline.add(&sink).unwrap(); + gstreamer::Element::link_many(&[last_element, &sink][..]).unwrap(); + + pipeline.set_state(gstreamer::State::Playing).unwrap(); + sink.sync_state_with_parent().unwrap(); + } + self.streams.push(stream.clone()); + } +} + +pub struct GstreamerMediaSocket { + proxy_sink: gstreamer::Element, +} + +impl GstreamerMediaSocket { + pub fn proxy_sink(&self) -> &gstreamer::Element { + &self.proxy_sink + } +} + +impl MediaSocket for GstreamerMediaSocket { + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/components/media/backends/gstreamer/media_stream_source.rs b/components/media/backends/gstreamer/media_stream_source.rs new file mode 100644 index 00000000000..36f0bdd55db --- /dev/null +++ b/components/media/backends/gstreamer/media_stream_source.rs @@ -0,0 +1,332 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use glib::subclass::prelude::*; +use gstreamer::prelude::*; +use gstreamer::subclass::prelude::*; +use gstreamer_base::UniqueFlowCombiner; +use once_cell::sync::Lazy; +use servo_media_streams::{MediaStream, MediaStreamType}; +use url::Url; + +use crate::media_stream::{GStreamerMediaStream, RTP_CAPS_OPUS, RTP_CAPS_VP8}; + +// Implementation sub-module of the GObject +mod imp { + use super::*; + + static AUDIO_SRC_PAD_TEMPLATE: Lazy = Lazy::new(|| { + gstreamer::PadTemplate::new( + "audio_src", + gstreamer::PadDirection::Src, + gstreamer::PadPresence::Sometimes, + &RTP_CAPS_OPUS, + ) + .expect("Could not create audio src pad template") + }); + + static VIDEO_SRC_PAD_TEMPLATE: Lazy = Lazy::new(|| { + gstreamer::PadTemplate::new( + "video_src", + gstreamer::PadDirection::Src, + gstreamer::PadPresence::Sometimes, + &RTP_CAPS_VP8, + ) + .expect("Could not create video src pad template") + }); + + pub struct ServoMediaStreamSrc { + cat: gstreamer::DebugCategory, + audio_proxysrc: gstreamer::Element, + audio_srcpad: gstreamer::GhostPad, + video_proxysrc: gstreamer::Element, + video_srcpad: gstreamer::GhostPad, + flow_combiner: Arc>, + has_audio_stream: Arc, + has_video_stream: Arc, + } + + impl ServoMediaStreamSrc { + pub fn set_stream( + &self, + stream: &mut GStreamerMediaStream, + src: &gstreamer::Element, + only_stream: bool, + ) { + // XXXferjm the current design limits the number of streams to one + // per type. This fulfills the basic use case for WebRTC, but we should + // implement support for multiple streams per type at some point, which + // likely involves encoding and muxing all streams of the same type + // in a single stream. + + gstreamer::log!(self.cat, "Setting stream"); + + // Append a proxysink to the media stream pipeline. + let pipeline = stream.pipeline_or_new(); + let last_element = stream.encoded(); + let sink = gstreamer::ElementFactory::make("proxysink") + .build() + .unwrap(); + pipeline.add(&sink).unwrap(); + gstreamer::Element::link_many(&[&last_element, &sink][..]).unwrap(); + + // Create the appropriate proxysrc depending on the stream type + // and connect the media stream proxysink to it. + self.setup_proxy_src(stream.ty(), &sink, src, only_stream); + + sink.sync_state_with_parent().unwrap(); + + pipeline.set_state(gstreamer::State::Playing).unwrap(); + } + + fn setup_proxy_src( + &self, + stream_type: MediaStreamType, + sink: &gstreamer::Element, + src: &gstreamer::Element, + only_stream: bool, + ) { + let (proxysrc, src_pad, no_more_pads) = match stream_type { + MediaStreamType::Audio => { + self.has_audio_stream.store(true, Ordering::Relaxed); + ( + &self.audio_proxysrc, + &self.audio_srcpad, + self.has_video_stream.load(Ordering::Relaxed), + ) + }, + MediaStreamType::Video => { + self.has_video_stream.store(true, Ordering::Relaxed); + ( + &self.video_proxysrc, + &self.video_srcpad, + self.has_audio_stream.load(Ordering::Relaxed), + ) + }, + }; + proxysrc.set_property("proxysink", sink); + + // Add proxysrc to bin + let bin = src.downcast_ref::().unwrap(); + bin.add(proxysrc) + .expect("Could not add proxysrc element to bin"); + + let target_pad = proxysrc + .static_pad("src") + .expect("Could not get proxysrc's static src pad"); + src_pad + .set_target(Some(&target_pad)) + .expect("Could not set target pad"); + + src.add_pad(src_pad) + .expect("Could not add source pad to media stream src"); + src.set_element_flags(gstreamer::ElementFlags::SOURCE); + + let proxy_pad = src_pad.internal().unwrap(); + src_pad.set_active(true).expect("Could not active pad"); + self.flow_combiner.lock().unwrap().add_pad(&proxy_pad); + + src.sync_state_with_parent().unwrap(); + + if no_more_pads || only_stream { + src.no_more_pads(); + } + } + } + + // Basic declaration of our type for the GObject type system. + #[glib::object_subclass] + impl ObjectSubclass for ServoMediaStreamSrc { + const NAME: &'static str = "ServoMediaStreamSrc"; + type Type = super::ServoMediaStreamSrc; + type ParentType = gstreamer::Bin; + type Interfaces = (gstreamer::URIHandler,); + + // Called once at the very beginning of instantiation of each instance and + // creates the data structure that contains all our state + fn with_class(_klass: &Self::Class) -> Self { + let flow_combiner = Arc::new(Mutex::new(UniqueFlowCombiner::new())); + + fn create_ghost_pad_with_template( + name: &str, + pad_template: &gstreamer::PadTemplate, + flow_combiner: Arc>, + ) -> gstreamer::GhostPad { + gstreamer::GhostPad::builder_from_template(pad_template) + .name(name) + .chain_function({ + move |pad, parent, buffer| { + let chain_result = + gstreamer::ProxyPad::chain_default(pad, parent, buffer); + let result = flow_combiner + .lock() + .unwrap() + .update_pad_flow(pad, chain_result); + if result == Err(gstreamer::FlowError::Flushing) { + return chain_result; + } + result + } + }) + .build() + } + + let audio_proxysrc = gstreamer::ElementFactory::make("proxysrc") + .build() + .expect("Could not create proxysrc element"); + let audio_srcpad = create_ghost_pad_with_template( + "audio_src", + &AUDIO_SRC_PAD_TEMPLATE, + flow_combiner.clone(), + ); + + let video_proxysrc = gstreamer::ElementFactory::make("proxysrc") + .build() + .expect("Could not create proxysrc element"); + let video_srcpad = create_ghost_pad_with_template( + "video_src", + &VIDEO_SRC_PAD_TEMPLATE, + flow_combiner.clone(), + ); + + Self { + cat: gstreamer::DebugCategory::new( + "servomediastreamsrc", + gstreamer::DebugColorFlags::empty(), + Some("Servo media stream source"), + ), + audio_proxysrc, + audio_srcpad, + video_proxysrc, + video_srcpad, + flow_combiner, + has_video_stream: Arc::new(AtomicBool::new(false)), + has_audio_stream: Arc::new(AtomicBool::new(false)), + } + } + } + + // The ObjectImpl trait provides the setters/getters for GObject properties. + // Here we need to provide the values that are internally stored back to the + // caller, or store whatever new value the caller is providing. + // + // This maps between the GObject properties and our internal storage of the + // corresponding values of the properties. + impl ObjectImpl for ServoMediaStreamSrc { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![ + // Let playbin3 know we are a live source. + glib::ParamSpecBoolean::builder("is-live") + .nick("Is Live") + .blurb("Let playbin3 know we are a live source") + .default_value(true) + .readwrite() + .build(), + ] + }); + + &PROPERTIES + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "is-live" => true.to_value(), + _ => unimplemented!(), + } + } + } + + impl GstObjectImpl for ServoMediaStreamSrc {} + + // Implementation of gstreamer::Element virtual methods + impl ElementImpl for ServoMediaStreamSrc { + fn metadata() -> Option<&'static gstreamer::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gstreamer::subclass::ElementMetadata::new( + "Servo Media Stream Source", + "Source/Audio/Video", + "Feed player with media stream data", + "Servo developers", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gstreamer::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + // Add pad templates for our audio and video source pads. + // These are later used for actually creating the pads and beforehand + // already provide information to GStreamer about all possible + // pads that could exist for this type. + vec![ + AUDIO_SRC_PAD_TEMPLATE.clone(), + VIDEO_SRC_PAD_TEMPLATE.clone(), + ] + }); + + PAD_TEMPLATES.as_ref() + } + } + + // Implementation of gstreamer::Bin virtual methods + impl BinImpl for ServoMediaStreamSrc {} + + impl URIHandlerImpl for ServoMediaStreamSrc { + const URI_TYPE: gstreamer::URIType = gstreamer::URIType::Src; + + fn protocols() -> &'static [&'static str] { + &["mediastream"] + } + + fn uri(&self) -> Option { + Some("mediastream://".to_string()) + } + + fn set_uri(&self, uri: &str) -> Result<(), glib::Error> { + if let Ok(uri) = Url::parse(uri) { + if uri.scheme() == "mediastream" { + return Ok(()); + } + } + Err(glib::Error::new( + gstreamer::URIError::BadUri, + format!("Invalid URI '{:?}'", uri,).as_str(), + )) + } + } +} + +// Public part of the ServoMediaStreamSrc type. This behaves like a normal +// GObject binding +glib::wrapper! { + pub struct ServoMediaStreamSrc(ObjectSubclass) + @extends gstreamer::Bin, gstreamer::Element, gstreamer::Object, @implements gstreamer::URIHandler; +} + +unsafe impl Send for ServoMediaStreamSrc {} +unsafe impl Sync for ServoMediaStreamSrc {} + +impl ServoMediaStreamSrc { + pub fn set_stream(&self, stream: &mut GStreamerMediaStream, only_stream: bool) { + self.imp() + .set_stream(stream, self.upcast_ref::(), only_stream) + } +} + +// Registers the type for our element, and then registers in GStreamer +// under the name "servomediastreamsrc" for being able to instantiate it via e.g. +// gstreamer::ElementFactory::make(). +pub fn register_servo_media_stream_src() -> Result<(), glib::BoolError> { + gstreamer::Element::register( + None, + "servomediastreamsrc", + gstreamer::Rank::NONE, + ServoMediaStreamSrc::static_type(), + ) +} diff --git a/components/media/backends/gstreamer/player.rs b/components/media/backends/gstreamer/player.rs new file mode 100644 index 00000000000..5b5c37dbbd9 --- /dev/null +++ b/components/media/backends/gstreamer/player.rs @@ -0,0 +1,1028 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::cell::{Cell, RefCell}; +use std::ops::Range; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Sender}; +use std::sync::{Arc, Mutex, Once}; +use std::time; + +use byte_slice_cast::AsSliceOf; +use glib::prelude::*; +use gstreamer_play::prelude::*; +use ipc_channel::ipc::{IpcReceiver, IpcSender, channel}; +use servo_media::MediaInstanceError; +use servo_media_player::audio::AudioRenderer; +use servo_media_player::context::PlayerGLContext; +use servo_media_player::metadata::Metadata; +use servo_media_player::video::VideoFrameRenderer; +use servo_media_player::{ + PlaybackState, Player, PlayerError, PlayerEvent, SeekLock, SeekLockMsg, StreamType, +}; +use servo_media_streams::registry::{MediaStreamId, get_stream}; +use servo_media_traits::{BackendMsg, ClientContextId, MediaInstance}; +use {glib, gstreamer, gstreamer_app, gstreamer_play}; + +use super::BACKEND_BASE_TIME; +use crate::media_stream::GStreamerMediaStream; +use crate::media_stream_source::{ServoMediaStreamSrc, register_servo_media_stream_src}; +use crate::render::GStreamerRender; +use crate::source::{ServoSrc, register_servo_src}; + +const DEFAULT_MUTED: bool = false; +const DEFAULT_PAUSED: bool = true; +const DEFAULT_CAN_RESUME: bool = false; +const DEFAULT_PLAYBACK_RATE: f64 = 1.0; +const DEFAULT_VOLUME: f64 = 1.0; +const DEFAULT_TIME_RANGES: Vec> = vec![]; + +const MAX_BUFFER_SIZE: i32 = 500 * 1024 * 1024; + +fn metadata_from_media_info(media_info: &gstreamer_play::PlayMediaInfo) -> Result { + let dur = media_info.duration(); + let duration = if let Some(dur) = dur { + let mut nanos = dur.nseconds(); + nanos %= 1_000_000_000; + let seconds = dur.seconds(); + Some(time::Duration::new(seconds, nanos as u32)) + } else { + None + }; + + let mut audio_tracks = Vec::new(); + let mut video_tracks = Vec::new(); + + let format = media_info + .container_format() + .unwrap_or_else(|| glib::GString::from("")) + .to_string(); + + for stream_info in media_info.stream_list() { + let stream_type = stream_info.stream_type(); + match stream_type.as_str() { + "audio" => { + let codec = stream_info + .codec() + .unwrap_or_else(|| glib::GString::from("")) + .to_string(); + audio_tracks.push(codec); + }, + "video" => { + let codec = stream_info + .codec() + .unwrap_or_else(|| glib::GString::from("")) + .to_string(); + video_tracks.push(codec); + }, + _ => {}, + } + } + + let mut width: u32 = 0; + let height: u32 = if media_info.number_of_video_streams() > 0 { + let first_video_stream = &media_info.video_streams()[0]; + width = first_video_stream.width() as u32; + first_video_stream.height() as u32 + } else { + 0 + }; + + let is_seekable = media_info.is_seekable(); + let is_live = media_info.is_live(); + let title = media_info.title().map(|s| s.as_str().to_string()); + + Ok(Metadata { + duration, + width, + height, + format, + is_seekable, + audio_tracks, + video_tracks, + is_live, + title, + }) +} + +pub struct GStreamerAudioChunk(gstreamer::buffer::MappedBuffer); +impl AsRef<[f32]> for GStreamerAudioChunk { + fn as_ref(&self) -> &[f32] { + self.0.as_ref().as_slice_of::().unwrap_or_default() + } +} + +#[derive(PartialEq)] +enum PlayerSource { + Seekable(ServoSrc), + Stream(ServoMediaStreamSrc), +} + +struct PlayerInner { + player: gstreamer_play::Play, + _signal_adapter: gstreamer_play::PlaySignalAdapter, + source: Option, + video_sink: gstreamer_app::AppSink, + input_size: u64, + play_state: gstreamer_play::PlayState, + paused: Cell, + can_resume: Cell, + playback_rate: Cell, + muted: Cell, + volume: Cell, + stream_type: StreamType, + last_metadata: Option, + cat: gstreamer::DebugCategory, + enough_data: Arc, +} + +impl PlayerInner { + pub fn set_input_size(&mut self, size: u64) -> Result<(), PlayerError> { + // Set input_size to proxy its value, since it + // could be set by the user before calling .setup(). + self.input_size = size; + if let Some(PlayerSource::Seekable(ref mut source)) = self.source { + source.set_size(if size > 0 { + size as i64 + } else { + -1 // live source + }); + } + Ok(()) + } + + pub fn set_mute(&mut self, muted: bool) -> Result<(), PlayerError> { + if self.muted.get() == muted { + return Ok(()); + } + + self.muted.set(muted); + self.player.set_mute(muted); + Ok(()) + } + + pub fn muted(&self) -> bool { + self.muted.get() + } + + pub fn set_playback_rate(&mut self, playback_rate: f64) -> Result<(), PlayerError> { + if self.stream_type != StreamType::Seekable { + return Err(PlayerError::NonSeekableStream); + } + + if self.playback_rate.get() == playback_rate { + return Ok(()); + } + + self.playback_rate.set(playback_rate); + + // The new playback rate will not be passed to the pipeline if the + // current GstPlay state is less than GST_STATE_PAUSED, which will be + // set immediately before the initial gstreamer_play_MESSAGE_MEDIA_INFO_UPDATED + // message is posted to bus. + if self.last_metadata.is_some() { + self.player.set_rate(playback_rate); + } + Ok(()) + } + + pub fn playback_rate(&self) -> f64 { + self.playback_rate.get() + } + + pub fn play(&mut self) -> Result<(), PlayerError> { + if !self.paused.get() { + return Ok(()); + } + + self.paused.set(false); + self.can_resume.set(false); + self.player.play(); + Ok(()) + } + + pub fn stop(&mut self) -> Result<(), PlayerError> { + self.player.stop(); + self.paused.set(true); + self.can_resume.set(false); + self.last_metadata = None; + self.source = None; + Ok(()) + } + + pub fn pause(&mut self) -> Result<(), PlayerError> { + if self.paused.get() { + return Ok(()); + } + + self.paused.set(true); + self.can_resume.set(true); + self.player.pause(); + Ok(()) + } + + pub fn paused(&self) -> bool { + self.paused.get() + } + + pub fn can_resume(&self) -> bool { + self.can_resume.get() + } + + pub fn end_of_stream(&mut self) -> Result<(), PlayerError> { + match self.source { + Some(ref mut source) => { + if let PlayerSource::Seekable(source) = source { + source + .push_end_of_stream() + .map(|_| ()) + .map_err(|_| PlayerError::EOSFailed) + } else { + Ok(()) + } + }, + _ => Ok(()), + } + } + + pub fn seek(&mut self, time: f64) -> Result<(), PlayerError> { + if self.stream_type != StreamType::Seekable { + return Err(PlayerError::NonSeekableStream); + } + if let Some(ref metadata) = self.last_metadata { + if let Some(ref duration) = metadata.duration { + if duration < &time::Duration::new(time as u64, 0) { + gstreamer::warning!( + self.cat, + obj = &self.player, + "Trying to seek out of range" + ); + return Err(PlayerError::SeekOutOfRange); + } + } + } + + let time = time * 1_000_000_000.; + self.player + .seek(gstreamer::ClockTime::from_nseconds(time as u64)); + Ok(()) + } + + pub fn set_volume(&mut self, volume: f64) -> Result<(), PlayerError> { + if self.volume.get() == volume { + return Ok(()); + } + + self.volume.set(volume); + self.player.set_volume(volume); + Ok(()) + } + + pub fn volume(&self) -> f64 { + self.volume.get() + } + + pub fn push_data(&mut self, data: Vec) -> Result<(), PlayerError> { + if let Some(PlayerSource::Seekable(ref mut source)) = self.source { + if self.enough_data.load(Ordering::Relaxed) { + return Err(PlayerError::EnoughData); + } + return source + .push_buffer(data) + .map(|_| ()) + .map_err(|_| PlayerError::BufferPushFailed); + } + Err(PlayerError::BufferPushFailed) + } + + pub fn set_src(&mut self, source: PlayerSource) { + self.source = Some(source); + } + + pub fn buffered(&self) -> Vec> { + let mut buffered_ranges = vec![]; + + let Some(duration) = self + .last_metadata + .as_ref() + .and_then(|metadata| metadata.duration) + else { + return buffered_ranges; + }; + + let pipeline = self.player.pipeline(); + let mut buffering = gstreamer::query::Buffering::new(gstreamer::Format::Percent); + if pipeline.query(&mut buffering) { + let ranges = buffering.ranges(); + for (start, end) in ranges { + let start = (if let gstreamer::GenericFormattedValue::Percent(start) = start { + start.unwrap() + } else { + gstreamer::format::Percent::from_percent(0) + } / gstreamer::format::Percent::MAX) as f64 * + duration.as_secs_f64(); + let end = (if let gstreamer::GenericFormattedValue::Percent(end) = end { + end.unwrap() + } else { + gstreamer::format::Percent::from_percent(0) + } / gstreamer::format::Percent::MAX) as f64 * + duration.as_secs_f64(); + buffered_ranges.push(Range { start, end }); + } + } + + buffered_ranges + } + + pub fn seekable(&self) -> Vec> { + // if the servosrc is seekable, we should return the duration of the media + if let Some(metadata) = self.last_metadata.as_ref() { + if metadata.is_seekable { + if let Some(duration) = metadata.duration { + return vec![Range { + start: 0.0, + end: duration.as_secs_f64(), + }]; + } + } + } + + // if the servosrc is not seekable, we should return the buffered range + self.buffered() + } + + fn set_stream(&mut self, stream: &MediaStreamId, only_stream: bool) -> Result<(), PlayerError> { + debug_assert!(self.stream_type == StreamType::Stream); + if let Some(PlayerSource::Stream(ref source)) = self.source { + let stream = + get_stream(stream).expect("Media streams registry does not contain such ID"); + let mut stream = stream.lock().unwrap(); + if let Some(stream) = stream.as_mut_any().downcast_mut::() { + let playbin = self + .player + .pipeline() + .dynamic_cast::() + .unwrap(); + let clock = gstreamer::SystemClock::obtain(); + playbin.set_base_time(*BACKEND_BASE_TIME); + playbin.set_start_time(gstreamer::ClockTime::NONE); + playbin.use_clock(Some(&clock)); + + source.set_stream(stream, only_stream); + return Ok(()); + } + } + Err(PlayerError::SetStreamFailed) + } + + fn set_audio_track(&mut self, stream_index: i32, enabled: bool) -> Result<(), PlayerError> { + self.player + .set_audio_track(stream_index) + .map_err(|_| PlayerError::SetTrackFailed)?; + self.player.set_audio_track_enabled(enabled); + Ok(()) + } + + fn set_video_track(&mut self, stream_index: i32, enabled: bool) -> Result<(), PlayerError> { + self.player + .set_video_track(stream_index) + .map_err(|_| PlayerError::SetTrackFailed)?; + self.player.set_video_track_enabled(enabled); + Ok(()) + } +} + +macro_rules! notify( + ($observer:expr_2021, $event:expr_2021) => { + $observer.lock().unwrap().send($event) + }; +); + +struct SeekChannel { + sender: SeekLock, + recv: IpcReceiver, +} + +impl SeekChannel { + fn new() -> Self { + let (sender, recv) = channel::().expect("Couldn't create IPC channel"); + Self { + sender: SeekLock { + lock_channel: sender, + }, + recv, + } + } + + fn sender(&self) -> SeekLock { + self.sender.clone() + } + + fn _await(&self) -> SeekLockMsg { + self.recv.recv().unwrap() + } +} + +pub struct GStreamerPlayer { + /// The player unique ID. + id: usize, + /// The ID of the client context this player belongs to. + context_id: ClientContextId, + /// Channel to communicate with the owner GStreamerBackend instance. + backend_chan: Arc>>, + inner: RefCell>>>, + observer: Arc>>, + audio_renderer: Option>>, + video_renderer: Option>>, + /// Indicates whether the setup was succesfully performed and + /// we are ready to consume a/v data. + is_ready: Arc, + /// Indicates whether the type of media stream to be played is a live stream. + stream_type: StreamType, + /// Decorator used to setup the video sink and process the produced frames. + render: Arc>, +} + +impl GStreamerPlayer { + #[allow(clippy::too_many_arguments)] + pub fn new( + id: usize, + context_id: &ClientContextId, + backend_chan: Arc>>, + stream_type: StreamType, + observer: IpcSender, + video_renderer: Option>>, + audio_renderer: Option>>, + gl_context: Box, + ) -> GStreamerPlayer { + let _ = gstreamer::DebugCategory::new( + "servoplayer", + gstreamer::DebugColorFlags::empty(), + Some("Servo player"), + ); + + Self { + id, + context_id: *context_id, + backend_chan, + inner: RefCell::new(None), + observer: Arc::new(Mutex::new(observer)), + audio_renderer, + video_renderer, + is_ready: Arc::new(Once::new()), + stream_type, + render: Arc::new(Mutex::new(GStreamerRender::new(gl_context))), + } + } + + fn setup(&self) -> Result<(), PlayerError> { + if self.inner.borrow().is_some() { + return Ok(()); + } + + // Check that we actually have the elements that we + // need to make this work. + for element in ["playbin3", "decodebin3", "queue"] { + if gstreamer::ElementFactory::find(element).is_none() { + return Err(PlayerError::Backend(format!( + "Missing dependency: {}", + element + ))); + } + } + + let player = gstreamer_play::Play::default(); + let signal_adapter = gstreamer_play::PlaySignalAdapter::new_sync_emit(&player); + let pipeline = player.pipeline(); + + // FIXME(#282): The progressive downloading breaks playback on Windows and Android. + if !cfg!(any(target_os = "windows", target_os = "android")) { + // Set player to perform progressive downloading. This will make the + // player store the downloaded media in a local temporary file for + // faster playback of already-downloaded chunks. + let flags = pipeline.property_value("flags"); + let flags_class = match glib::FlagsClass::with_type(flags.type_()) { + Some(flags) => flags, + None => { + return Err(PlayerError::Backend( + "FlagsClass creation failed".to_owned(), + )); + }, + }; + let flags_class = match flags_class.builder_with_value(flags) { + Some(class) => class, + None => { + return Err(PlayerError::Backend( + "FlagsClass creation failed".to_owned(), + )); + }, + }; + let Some(flags) = flags_class.set_by_nick("download").build() else { + return Err(PlayerError::Backend( + "FlagsClass creation failed".to_owned(), + )); + }; + pipeline.set_property_from_value("flags", &flags); + } + + // Set max size for the player buffer. + pipeline.set_property("buffer-size", MAX_BUFFER_SIZE); + + // Set player position interval update to 0.5 seconds. + let mut config = player.config(); + config.set_position_update_interval(500u32); + player + .set_config(config) + .map_err(|e| PlayerError::Backend(e.to_string()))?; + + if let Some(ref audio_renderer) = self.audio_renderer { + let audio_sink = + gstreamer::ElementFactory::make("appsink") + .build() + .map_err(|error| { + PlayerError::Backend(format!("appsink creation failed: {error:?}")) + })?; + + pipeline.set_property("audio-sink", &audio_sink); + + let audio_sink = audio_sink.dynamic_cast::().unwrap(); + + let weak_audio_renderer = Arc::downgrade(audio_renderer); + + audio_sink.set_callbacks( + gstreamer_app::AppSinkCallbacks::builder() + .new_preroll(|_| Ok(gstreamer::FlowSuccess::Ok)) + .new_sample(move |audio_sink| { + let sample = audio_sink + .pull_sample() + .map_err(|_| gstreamer::FlowError::Eos)?; + let buffer = sample.buffer_owned().ok_or(gstreamer::FlowError::Error)?; + let audio_info = sample + .caps() + .and_then(|caps| gstreamer_audio::AudioInfo::from_caps(caps).ok()) + .ok_or(gstreamer::FlowError::Error)?; + let positions = + audio_info.positions().ok_or(gstreamer::FlowError::Error)?; + + let Some(audio_renderer) = weak_audio_renderer.upgrade() else { + return Err(gstreamer::FlowError::Flushing); + }; + + for position in positions.iter() { + let buffer = buffer.clone(); + let map = match buffer.into_mapped_buffer_readable() { + Ok(map) => map, + _ => { + return Err(gstreamer::FlowError::Error); + }, + }; + let chunk = Box::new(GStreamerAudioChunk(map)); + let channel = position.to_mask() as u32; + + audio_renderer.lock().unwrap().render(chunk, channel); + } + Ok(gstreamer::FlowSuccess::Ok) + }) + .build(), + ); + } + + let video_sink = self.render.lock().unwrap().setup_video_sink(&pipeline)?; + + // There's a known bug in gstreamer that may cause a wrong transition + // to the ready state while setting the uri property: + // https://cgit.freedesktop.org/gstreamer/gst-plugins-bad/commit/?id=afbbc3a97ec391c6a582f3c746965fdc3eb3e1f3 + // This may affect things like setting the config, so until the bug is + // fixed, make sure that state dependent code happens before this line. + // The estimated version for the fix is 1.14.5 / 1.15.1. + // https://github.com/servo/servo/issues/22010#issuecomment-432599657 + let uri = match self.stream_type { + StreamType::Stream => { + register_servo_media_stream_src().map_err(|error| { + PlayerError::Backend(format!( + "servomediastreamsrc registration error: {error:?}" + )) + })?; + "mediastream://".to_value() + }, + StreamType::Seekable => { + register_servo_src().map_err(|error| { + PlayerError::Backend(format!("servosrc registration error: {error:?}")) + })?; + "servosrc://".to_value() + }, + }; + player.set_property("uri", &uri); + + // No video_renderers no video + if self.video_renderer.is_none() { + player.set_video_track_enabled(false); + } + + *self.inner.borrow_mut() = Some(Arc::new(Mutex::new(PlayerInner { + player, + _signal_adapter: signal_adapter.clone(), + source: None, + video_sink, + input_size: 0, + play_state: gstreamer_play::PlayState::Stopped, + paused: Cell::new(DEFAULT_PAUSED), + can_resume: Cell::new(DEFAULT_CAN_RESUME), + playback_rate: Cell::new(DEFAULT_PLAYBACK_RATE), + muted: Cell::new(DEFAULT_MUTED), + volume: Cell::new(DEFAULT_VOLUME), + stream_type: self.stream_type, + last_metadata: None, + cat: gstreamer::DebugCategory::get("servoplayer").unwrap(), + enough_data: Arc::new(AtomicBool::new(false)), + }))); + + let inner = self.inner.borrow(); + let inner = inner.as_ref().unwrap(); + let observer = self.observer.clone(); + // Handle `end-of-stream` signal. + signal_adapter.connect_end_of_stream(move |_| { + let _ = notify!(observer, PlayerEvent::EndOfStream); + }); + + let observer = self.observer.clone(); + // Handle `error` signal + signal_adapter.connect_error(move |_self, error, _details| { + let _ = notify!(observer, PlayerEvent::Error(error.to_string())); + }); + + let inner_clone = inner.clone(); + let observer = self.observer.clone(); + // Handle `state-changed` signal. + signal_adapter.connect_state_changed(move |_, play_state| { + inner_clone.lock().unwrap().play_state = play_state; + + let state = match play_state { + gstreamer_play::PlayState::Buffering => Some(PlaybackState::Buffering), + gstreamer_play::PlayState::Stopped => Some(PlaybackState::Stopped), + gstreamer_play::PlayState::Paused => Some(PlaybackState::Paused), + gstreamer_play::PlayState::Playing => Some(PlaybackState::Playing), + _ => None, + }; + if let Some(v) = state { + let _ = notify!(observer, PlayerEvent::StateChanged(v)); + } + }); + + let observer = self.observer.clone(); + // Handle `position-update` signal. + signal_adapter.connect_position_updated(move |_, position| { + if let Some(seconds) = position.map(|p| p.seconds_f64()) { + let _ = notify!(observer, PlayerEvent::PositionChanged(seconds)); + } + }); + + let observer = self.observer.clone(); + // Handle `seek-done` signal. + signal_adapter.connect_seek_done(move |_, position| { + let _ = notify!(observer, PlayerEvent::SeekDone(position.seconds_f64())); + }); + + // Handle `media-info-updated` signal. + let inner_clone = inner.clone(); + let observer = self.observer.clone(); + signal_adapter.connect_media_info_updated(move |_, info| { + let Ok(metadata) = metadata_from_media_info(info) else { + return; + }; + + let mut inner = inner_clone.lock().unwrap(); + + if inner.last_metadata.as_ref() == Some(&metadata) { + return; + } + + // TODO: Workaround to generate expected `paused` state change event. + // + let mut send_pause_event = false; + + if inner.last_metadata.is_none() && metadata.is_seekable { + if inner.playback_rate.get() != DEFAULT_PLAYBACK_RATE { + // The `paused` state change event will be fired after the + // seek initiated by the playback rate change has + // completed. + inner.player.set_rate(inner.playback_rate.get()); + } else if inner.play_state == gstreamer_play::PlayState::Paused { + send_pause_event = true; + } + } + + inner.last_metadata = Some(metadata.clone()); + gstreamer::info!( + inner.cat, + obj = &inner.player, + "Metadata updated: {:?}", + metadata + ); + let _ = notify!(observer, PlayerEvent::MetadataUpdated(metadata)); + + if send_pause_event { + let _ = notify!(observer, PlayerEvent::StateChanged(PlaybackState::Paused)); + } + }); + + // Handle `duration-changed` signal. + let inner_clone = inner.clone(); + let observer = self.observer.clone(); + signal_adapter.connect_duration_changed(move |_, duration| { + let duration = duration.map(|duration| { + time::Duration::new( + duration.seconds(), + (duration.nseconds() % 1_000_000_000) as u32, + ) + }); + + let mut inner = inner_clone.lock().unwrap(); + if let Some(ref mut metadata) = inner.last_metadata { + if metadata.duration != duration { + metadata.duration = duration; + gstreamer::info!( + inner.cat, + obj = &inner.player, + "Duration changed: {:?}", + duration + ); + let _ = notify!(observer, PlayerEvent::DurationChanged(duration)); + } + } + }); + + if let Some(video_renderer) = self.video_renderer.clone() { + // Creates a closure that renders a frame using the video_renderer + // Used in the preroll and sample callbacks + let render_sample = { + let render = self.render.clone(); + let observer = self.observer.clone(); + let weak_video_renderer = Arc::downgrade(&video_renderer); + + move |sample: gstreamer::Sample| { + let Some(frame) = render.lock().unwrap().get_frame_from_sample(sample) else { + return Err(gstreamer::FlowError::Error); + }; + + match weak_video_renderer.upgrade() { + Some(video_renderer) => { + video_renderer.lock().unwrap().render(frame); + }, + _ => { + return Err(gstreamer::FlowError::Flushing); + }, + }; + + let _ = notify!(observer, PlayerEvent::VideoFrameUpdated); + Ok(gstreamer::FlowSuccess::Ok) + } + }; + + // Set video_sink callbacks. + inner.lock().unwrap().video_sink.set_callbacks( + gstreamer_app::AppSinkCallbacks::builder() + .new_preroll({ + let render_sample = render_sample.clone(); + move |video_sink| { + render_sample( + video_sink + .pull_preroll() + .map_err(|_| gstreamer::FlowError::Eos)?, + ) + } + }) + .new_sample(move |video_sink| { + render_sample( + video_sink + .pull_sample() + .map_err(|_| gstreamer::FlowError::Eos)?, + ) + }) + .build(), + ); + }; + + let (receiver, error_handler_id) = { + let inner_clone = inner.clone(); + let inner = inner.lock().unwrap(); + let pipeline = inner.player.pipeline(); + + let (sender, receiver) = mpsc::channel(); + + let sender = Arc::new(Mutex::new(sender)); + let sender_clone = sender.clone(); + let is_ready_clone = self.is_ready.clone(); + let observer = self.observer.clone(); + pipeline.connect("source-setup", false, move |args| { + let source = args[1].get::().unwrap(); + + let mut inner = inner_clone.lock().unwrap(); + let source = match inner.stream_type { + StreamType::Seekable => { + let servosrc = source + .dynamic_cast::() + .expect("Source element is expected to be a ServoSrc!"); + + if inner.input_size > 0 { + servosrc.set_size(inner.input_size as i64); + } + + let sender_clone = sender.clone(); + let is_ready = is_ready_clone.clone(); + let observer_ = observer.clone(); + let observer__ = observer.clone(); + let observer___ = observer.clone(); + let servosrc_ = servosrc.clone(); + let enough_data_ = inner.enough_data.clone(); + let enough_data__ = inner.enough_data.clone(); + let seek_channel = Arc::new(Mutex::new(SeekChannel::new())); + servosrc.set_callbacks( + gstreamer_app::AppSrcCallbacks::builder() + .need_data(move |_, _| { + // We block the caller of the setup method until we get + // the first need-data signal, so we ensure that we + // don't miss any data between the moment the client + // calls setup and the player is actually ready to + // get any data. + is_ready.call_once(|| { + let _ = sender_clone.lock().unwrap().send(Ok(())); + }); + + enough_data_.store(false, Ordering::Relaxed); + let _ = notify!(observer_, PlayerEvent::NeedData); + }) + .enough_data(move |_| { + enough_data__.store(true, Ordering::Relaxed); + let _ = notify!(observer__, PlayerEvent::EnoughData); + }) + .seek_data(move |_, offset| { + let (ret, ack_channel) = if servosrc_.set_seek_offset(offset) { + let _ = notify!( + observer___, + PlayerEvent::SeekData( + offset, + seek_channel.lock().unwrap().sender() + ) + ); + let (ret, ack_channel) = + seek_channel.lock().unwrap()._await(); + (ret, Some(ack_channel)) + } else { + (true, None) + }; + + servosrc_.set_seek_done(); + if let Some(ack_channel) = ack_channel { + ack_channel.send(()).unwrap(); + } + ret + }) + .build(), + ); + + PlayerSource::Seekable(servosrc) + }, + StreamType::Stream => { + let media_stream_src = source + .dynamic_cast::() + .expect("Source element is expected to be a ServoMediaStreamSrc!"); + let sender_clone = sender.clone(); + is_ready_clone.call_once(|| { + let _ = notify!(sender_clone, Ok(())); + }); + PlayerSource::Stream(media_stream_src) + }, + }; + + inner.set_src(source); + + None + }); + + let error_handler_id = + signal_adapter.connect_error(move |signal_adapter, error, _details| { + let _ = notify!(sender_clone, Err(PlayerError::Backend(error.to_string()))); + signal_adapter.play().stop(); + }); + + inner.player.pause(); + + (receiver, error_handler_id) + }; + + let result = receiver.recv().unwrap(); + glib::signal::signal_handler_disconnect(&inner.lock().unwrap().player, error_handler_id); + result + } +} + +macro_rules! inner_player_proxy_getter { + ($fn_name:ident, $return_type:ty, $default_value:expr_2021) => { + fn $fn_name(&self) -> $return_type { + if self.setup().is_err() { + return $default_value; + } + + let inner = self.inner.borrow(); + let inner = inner.as_ref().unwrap().lock().unwrap(); + inner.$fn_name() + } + }; +} + +macro_rules! inner_player_proxy { + ($fn_name:ident, $return_type:ty) => { + fn $fn_name(&self) -> Result<$return_type, PlayerError> { + self.setup()?; + let inner = self.inner.borrow(); + let mut inner = inner.as_ref().unwrap().lock().unwrap(); + inner.$fn_name() + } + }; + + ($fn_name:ident, $arg1:ident, $arg1_type:ty) => { + fn $fn_name(&self, $arg1: $arg1_type) -> Result<(), PlayerError> { + self.setup()?; + let inner = self.inner.borrow(); + let mut inner = inner.as_ref().unwrap().lock().unwrap(); + inner.$fn_name($arg1) + } + }; + + ($fn_name:ident, $arg1:ident, $arg1_type:ty, $arg2:ident, $arg2_type:ty) => { + fn $fn_name(&self, $arg1: $arg1_type, $arg2: $arg2_type) -> Result<(), PlayerError> { + self.setup()?; + let inner = self.inner.borrow(); + let mut inner = inner.as_ref().unwrap().lock().unwrap(); + inner.$fn_name($arg1, $arg2) + } + }; +} + +impl Player for GStreamerPlayer { + inner_player_proxy!(play, ()); + inner_player_proxy!(pause, ()); + inner_player_proxy_getter!(paused, bool, DEFAULT_PAUSED); + inner_player_proxy_getter!(can_resume, bool, DEFAULT_CAN_RESUME); + inner_player_proxy!(stop, ()); + inner_player_proxy!(end_of_stream, ()); + inner_player_proxy!(set_input_size, size, u64); + inner_player_proxy!(set_mute, muted, bool); + inner_player_proxy_getter!(muted, bool, DEFAULT_MUTED); + inner_player_proxy!(set_playback_rate, playback_rate, f64); + inner_player_proxy_getter!(playback_rate, f64, DEFAULT_PLAYBACK_RATE); + inner_player_proxy!(push_data, data, Vec); + inner_player_proxy!(seek, time, f64); + inner_player_proxy!(set_volume, volume, f64); + inner_player_proxy_getter!(volume, f64, DEFAULT_VOLUME); + inner_player_proxy_getter!(buffered, Vec>, DEFAULT_TIME_RANGES); + inner_player_proxy_getter!(seekable, Vec>, DEFAULT_TIME_RANGES); + inner_player_proxy!(set_stream, stream, &MediaStreamId, only_stream, bool); + inner_player_proxy!(set_audio_track, stream_index, i32, enabled, bool); + inner_player_proxy!(set_video_track, stream_index, i32, enabled, bool); + + fn render_use_gl(&self) -> bool { + self.render.lock().unwrap().is_gl() + } +} + +impl MediaInstance for GStreamerPlayer { + fn get_id(&self) -> usize { + self.id + } + + fn mute(&self, val: bool) -> Result<(), MediaInstanceError> { + self.set_mute(val).map_err(|_| MediaInstanceError) + } + + fn suspend(&self) -> Result<(), MediaInstanceError> { + self.pause().map_err(|_| MediaInstanceError) + } + + fn resume(&self) -> Result<(), MediaInstanceError> { + if !self.can_resume() { + return Ok(()); + } + + self.play().map_err(|_| MediaInstanceError) + } +} + +impl Drop for GStreamerPlayer { + fn drop(&mut self) { + let _ = self.stop(); + let (tx_ack, rx_ack) = mpsc::channel(); + let _ = self + .backend_chan + .lock() + .unwrap() + .send(BackendMsg::Shutdown { + context: self.context_id, + id: self.id, + tx_ack, + }); + let _ = rx_ack.recv(); + } +} diff --git a/components/media/backends/gstreamer/registry_scanner.rs b/components/media/backends/gstreamer/registry_scanner.rs new file mode 100644 index 00000000000..62e4145551d --- /dev/null +++ b/components/media/backends/gstreamer/registry_scanner.rs @@ -0,0 +1,266 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::collections::HashSet; +use std::str::FromStr; + +use once_cell::sync::Lazy; + +// The GStreamer registry holds the metadata of the set of plugins available in the host. +// This scanner is used to lazily analyze the registry and to provide information about +// the set of supported mime types and codecs that the backend is able to deal with. +pub static GSTREAMER_REGISTRY_SCANNER: Lazy = + Lazy::new(GStreamerRegistryScanner::new); + +pub struct GStreamerRegistryScanner { + supported_mime_types: HashSet<&'static str>, + supported_codecs: HashSet<&'static str>, +} + +impl GStreamerRegistryScanner { + fn new() -> GStreamerRegistryScanner { + let mut registry_scanner = GStreamerRegistryScanner { + supported_mime_types: HashSet::new(), + supported_codecs: HashSet::new(), + }; + registry_scanner.initialize(); + registry_scanner + } + + pub fn is_container_type_supported(&self, container_type: &str) -> bool { + self.supported_mime_types.contains(container_type) + } + + fn is_codec_supported(&self, codec: &str) -> bool { + self.supported_codecs.contains(codec) + } + + pub fn are_all_codecs_supported(&self, codecs: &Vec<&str>) -> bool { + codecs.iter().all(|&codec| self.is_codec_supported(codec)) + } + + fn initialize(&mut self) { + let audio_decoder_factories = gstreamer::ElementFactory::factories_with_type( + gstreamer::ElementFactoryType::DECODER | gstreamer::ElementFactoryType::MEDIA_AUDIO, + gstreamer::Rank::MARGINAL, + ); + let audio_parser_factories = gstreamer::ElementFactory::factories_with_type( + gstreamer::ElementFactoryType::PARSER | gstreamer::ElementFactoryType::MEDIA_AUDIO, + gstreamer::Rank::NONE, + ); + let video_decoder_factories = gstreamer::ElementFactory::factories_with_type( + gstreamer::ElementFactoryType::DECODER | gstreamer::ElementFactoryType::MEDIA_VIDEO, + gstreamer::Rank::MARGINAL, + ); + let video_parser_factories = gstreamer::ElementFactory::factories_with_type( + gstreamer::ElementFactoryType::PARSER | gstreamer::ElementFactoryType::MEDIA_VIDEO, + gstreamer::Rank::MARGINAL, + ); + let demux_factories = gstreamer::ElementFactory::factories_with_type( + gstreamer::ElementFactoryType::DEMUXER, + gstreamer::Rank::MARGINAL, + ); + + if has_element_for_media_type(&audio_decoder_factories, "audio/mpeg, mpegversion=(int)4") { + self.supported_mime_types.insert("audio/aac"); + self.supported_mime_types.insert("audio/mp4"); + self.supported_mime_types.insert("audio/x-m4a"); + self.supported_codecs.insert("mpeg"); + self.supported_codecs.insert("mp4a*"); + } + + let is_opus_supported = + has_element_for_media_type(&audio_decoder_factories, "audio/x-opus"); + if is_opus_supported && has_element_for_media_type(&audio_parser_factories, "audio/x-opus") + { + self.supported_mime_types.insert("audio/opus"); + self.supported_codecs.insert("opus"); + self.supported_codecs.insert("x-opus"); + } + + let is_vorbis_supported = + has_element_for_media_type(&audio_decoder_factories, "audio/x-vorbis"); + if is_vorbis_supported && + has_element_for_media_type(&audio_parser_factories, "audio/x-vorbis") + { + self.supported_codecs.insert("vorbis"); + self.supported_codecs.insert("x-vorbis"); + } + + if has_element_for_media_type(&demux_factories, "video/x-matroska") { + let is_vp8_decoder_available = + has_element_for_media_type(&video_decoder_factories, "video/x-vp8"); + let is_vp9_decoder_available = + has_element_for_media_type(&video_decoder_factories, "video/x-vp9"); + + if is_vp8_decoder_available || is_vp9_decoder_available { + self.supported_mime_types.insert("video/webm"); + } + + if is_vp8_decoder_available { + self.supported_codecs.insert("vp8"); + self.supported_codecs.insert("x-vp8"); + self.supported_codecs.insert("vp8.0"); + } + + if is_vp9_decoder_available { + self.supported_codecs.insert("vp9"); + self.supported_codecs.insert("x-vp9"); + self.supported_codecs.insert("vp9.0"); + } + + if is_opus_supported { + self.supported_mime_types.insert("audio/webm"); + } + } + + let is_h264_decoder_available = has_element_for_media_type( + &video_decoder_factories, + "video/x-h264, profile=(string){ constrained-baseline, baseline, high }", + ); + if is_h264_decoder_available && + has_element_for_media_type(&video_parser_factories, "video/x-h264") + { + self.supported_mime_types.insert("video/mp4"); + self.supported_mime_types.insert("video/x-m4v"); + self.supported_codecs.insert("x-h264"); + self.supported_codecs.insert("avc*"); + self.supported_codecs.insert("mp4v*"); + } + + if has_element_for_media_type(&audio_decoder_factories, "audio/midi") { + self.supported_mime_types.insert("audio/midi"); + self.supported_mime_types.insert("audio/riff-midi"); + } + + if has_element_for_media_type(&audio_decoder_factories, "audio/x-ac3") { + self.supported_mime_types.insert("audio/x-ac3"); + } + + if has_element_for_media_type(&audio_decoder_factories, "audio/x-flac") { + self.supported_mime_types.insert("audio/flac"); + self.supported_mime_types.insert("audio/x-flac"); + } + + if has_element_for_media_type(&audio_decoder_factories, "audio/x-speex") { + self.supported_mime_types.insert("audio/speex"); + self.supported_mime_types.insert("audio/x-speex"); + } + + if has_element_for_media_type(&audio_decoder_factories, "audio/x-wavpack") { + self.supported_mime_types.insert("audio/x-wavpack"); + } + + if has_element_for_media_type( + &video_decoder_factories, + "video/mpeg, mpegversion=(int){1,2}, systemstream=(boolean)false", + ) { + self.supported_mime_types.insert("video/mpeg"); + self.supported_codecs.insert("mpeg"); + } + + if has_element_for_media_type(&video_decoder_factories, "video/x-flash-video") { + self.supported_mime_types.insert("video/flv"); + self.supported_mime_types.insert("video/x-flv"); + } + + if has_element_for_media_type(&video_decoder_factories, "video/x-msvideocodec") { + self.supported_mime_types.insert("video/x-msvideo"); + } + + if has_element_for_media_type(&demux_factories, "application/x-hls") { + self.supported_mime_types + .insert("application/vnd.apple.mpegurl"); + self.supported_mime_types.insert("application/x-mpegurl"); + } + + if has_element_for_media_type(&demux_factories, "application/x-wav") || + has_element_for_media_type(&demux_factories, "audio/x-wav") + { + self.supported_mime_types.insert("audio/wav"); + self.supported_mime_types.insert("audio/vnd.wav"); + self.supported_mime_types.insert("audio/x-wav"); + self.supported_codecs.insert("1"); + } + + if has_element_for_media_type(&demux_factories, "video/quicktime, variant=(string)3gpp") { + self.supported_mime_types.insert("video/3gpp"); + } + + if has_element_for_media_type(&demux_factories, "application/ogg") { + self.supported_mime_types.insert("application/ogg"); + + if is_vorbis_supported { + self.supported_mime_types.insert("audio/ogg"); + self.supported_mime_types.insert("audio/x-vorbis+ogg"); + } + + if has_element_for_media_type(&audio_decoder_factories, "audio/x-speex") { + self.supported_mime_types.insert("audio/ogg"); + self.supported_codecs.insert("speex"); + } + + if has_element_for_media_type(&video_decoder_factories, "video/x-theora") { + self.supported_mime_types.insert("video/ogg"); + self.supported_codecs.insert("theora"); + } + } + + let mut is_audio_mpeg_supported = false; + if has_element_for_media_type( + &audio_decoder_factories, + "audio/mpeg, mpegversion=(int)1, layer=(int)[1, 3]", + ) { + is_audio_mpeg_supported = true; + self.supported_mime_types.insert("audio/mp1"); + self.supported_mime_types.insert("audio/mp3"); + self.supported_mime_types.insert("audio/x-mp3"); + self.supported_codecs.insert("audio/mp3"); + } + + if has_element_for_media_type(&audio_decoder_factories, "audio/mpeg, mpegversion=(int)2") { + is_audio_mpeg_supported = true; + self.supported_mime_types.insert("audio/mp2"); + } + + is_audio_mpeg_supported |= self.is_container_type_supported("video/mp4"); + if is_audio_mpeg_supported { + self.supported_mime_types.insert("audio/mpeg"); + self.supported_mime_types.insert("audio/x-mpeg"); + } + + let is_matroska_supported = + has_element_for_media_type(&demux_factories, "video/x-matroska"); + if is_matroska_supported { + self.supported_mime_types.insert("video/x-matroska"); + + if has_element_for_media_type(&video_decoder_factories, "video/x-vp10") { + self.supported_mime_types.insert("video/webm"); + } + } + + if (is_matroska_supported || self.is_container_type_supported("video/mp4")) && + has_element_for_media_type(&video_decoder_factories, "video/x-av1") + { + self.supported_codecs.insert("av01*"); + } + } +} + +fn has_element_for_media_type( + factories: &glib::List, + media_type: &str, +) -> bool { + match gstreamer::caps::Caps::from_str(media_type) { + Ok(caps) => { + for factory in factories { + if factory.can_sink_all_caps(&caps) { + return true; + } + } + false + }, + _ => false, + } +} diff --git a/components/media/backends/gstreamer/render-android/Cargo.toml b/components/media/backends/gstreamer/render-android/Cargo.toml new file mode 100644 index 00000000000..a7e1291b5a9 --- /dev/null +++ b/components/media/backends/gstreamer/render-android/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "servo-media-gstreamer-render-android" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_gstreamer_render_android" +path = "lib.rs" + +[dependencies] +glib = { workspace = true } +gstreamer = { workspace = true } +gstreamer-gl = { workspace = true } +gstreamer-gl-egl = { workspace = true } +gstreamer-video = { workspace = true } +sm-player = { package = "servo-media-player", path = "../../../player" } +sm-gst-render = { package = "servo-media-gstreamer-render", path = "../render" } diff --git a/components/media/backends/gstreamer/render-android/lib.rs b/components/media/backends/gstreamer/render-android/lib.rs new file mode 100644 index 00000000000..da9a8395d37 --- /dev/null +++ b/components/media/backends/gstreamer/render-android/lib.rs @@ -0,0 +1,256 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +//! `RenderAndroid` is a `Render` implementation for Android +//! platform. It only implements an OpenGLES mechanism. +//! +//! Internally it uses GStreamer's *glsinkbin* element as *videosink* +//! wrapping the *appsink* from the Player. And the shared frames are +//! mapped as texture IDs. + +use std::sync::{Arc, Mutex}; + +use gstreamer::prelude::*; +use gstreamer_gl::prelude::*; +use sm_gst_render::Render; +use sm_player::PlayerError; +use sm_player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; +use sm_player::video::{Buffer, VideoFrame, VideoFrameData}; + +struct GStreamerBuffer { + is_external_oes: bool, + frame: gstreamer_gl::GLVideoFrame, +} + +impl Buffer for GStreamerBuffer { + fn to_vec(&self) -> Result { + // packed formats are guaranteed to be in a single plane + if self.frame.format() == gstreamer_video::VideoFormat::Rgba { + let tex_id = self.frame.texture_id(0).map_err(|_| ())?; + Ok(if self.is_external_oes { + VideoFrameData::OESTexture(tex_id) + } else { + VideoFrameData::Texture(tex_id) + }) + } else { + Err(()) + } + } +} + +pub struct RenderAndroid { + display: gstreamer_gl::GLDisplay, + app_context: gstreamer_gl::GLContext, + gst_context: Arc>>, + gl_upload: Arc>>, +} + +impl RenderAndroid { + /// Tries to create a new intance of the `RenderAndroid` + /// + /// # Arguments + /// + /// * `context` - is the PlayerContext trait object from + /// application. + pub fn new(app_gl_context: Box) -> Option { + // Check that we actually have the elements that we + // need to make this work. + if gstreamer::ElementFactory::find("glsinkbin").is_none() { + return None; + } + + let display_native = app_gl_context.get_native_display(); + let gl_context = app_gl_context.get_gl_context(); + let gl_api = match app_gl_context.get_gl_api() { + GlApi::OpenGL => gstreamer_gl::GLAPI::OPENGL, + GlApi::OpenGL3 => gstreamer_gl::GLAPI::OPENGL3, + GlApi::Gles1 => gstreamer_gl::GLAPI::GLES1, + GlApi::Gles2 => gstreamer_gl::GLAPI::GLES2, + GlApi::None => return None, + }; + + let (wrapped_context, display) = match gl_context { + GlContext::Egl(context) => { + let display = match display_native { + NativeDisplay::Egl(display_native) => { + unsafe { gstreamer_gl_egl::GLDisplayEGL::with_egl_display(display_native) } + .and_then(|display| Ok(display.upcast())) + .ok() + }, + _ => None, + }; + + match display { + Some(display) => { + let wrapped_context = unsafe { + gstreamer_gl::GLContext::new_wrapped( + &display, + context, + gstreamer_gl::GLPlatform::EGL, + gl_api, + ) + }; + (wrapped_context, Some(display)) + }, + _ => (None, None), + } + }, + _ => (None, None), + }; + + match wrapped_context { + Some(app_context) => Some(RenderAndroid { + display: display.unwrap(), + app_context, + gst_context: Arc::new(Mutex::new(None)), + gl_upload: Arc::new(Mutex::new(None)), + }), + _ => None, + } + } +} + +impl Render for RenderAndroid { + fn is_gl(&self) -> bool { + true + } + + fn build_frame(&self, sample: gstreamer::Sample) -> Option { + if self.gst_context.lock().unwrap().is_none() && self.gl_upload.lock().unwrap().is_some() { + *self.gst_context.lock().unwrap() = match self.gl_upload.lock().unwrap().as_ref() { + Some(glupload) => Some(glupload.property::("context")), + _ => None, + }; + } + + let buffer = sample.buffer_owned().ok_or_else(|| ())?; + let caps = sample.caps().ok_or_else(|| ())?; + + let is_external_oes = caps + .structure(0) + .and_then(|s| { + s.get::<&str>("texture-target").ok().and_then(|target| { + if target == "external-oes" { + Some(s) + } else { + None + } + }) + }) + .is_some(); + + let info = gstreamer_video::VideoInfo::from_caps(caps).ok()?; + + if self.gst_context.lock().unwrap().is_some() { + if let Some(sync_meta) = buffer.meta::() { + sync_meta.set_sync_point(self.gst_context.lock().unwrap().as_ref().unwrap()); + } + } + + let frame = + gstreamer_gl::GLVideoFrame::from_buffer_readable(buffer, &info).or_else(|_| Err(()))?; + + if self.gst_context.lock().unwrap().is_some() { + if let Some(sync_meta) = frame.buffer().meta::() { + // This should possibly be + // sync_meta.wait(&self.app_context); + // since we want the main app thread to sync it's GPU pipeline too, + // but the main thread and the app context aren't managed by gstreamer, + // so we can't do that directly. + // https://github.com/servo/media/issues/309 + sync_meta.wait(self.gst_context.lock().unwrap().as_ref().unwrap()); + } + } + + VideoFrame::new( + info.width() as i32, + info.height() as i32, + Arc::new(GStreamerBuffer { + is_external_oes, + frame, + }), + ) + } + + fn build_video_sink( + &self, + appsink: &gstreamer::Element, + pipeline: &gstreamer::Element, + ) -> Result<(), PlayerError> { + if self.gl_upload.lock().unwrap().is_some() { + return Err(PlayerError::Backend( + "render unix already setup the video sink".to_owned(), + )); + } + + let caps = gstreamer::Caps::builder("video/x-raw") + .features([gstreamer_gl::CAPS_FEATURE_MEMORY_GL_MEMORY]) + .field("format", gstreamer_video::VideoFormat::Rgba.to_str()) + .field( + "texture-target", + gstreamer::List::new(["2D", "external-oes"]), + ) + .build(); + appsink.set_property("caps", &caps); + + let vsinkbin = gstreamer::ElementFactory::make("glsinkbin") + .name("servo-media-vsink") + .property("sink", &appsink) + .build() + .map_err(|error| { + PlayerError::Backend(format!("glupload creation failed: {error:?}")) + })?; + + pipeline.set_property("video-sink", &vsinkbin); + + let bus = pipeline.bus().expect("pipeline with no bus"); + let display_ = self.display.clone(); + let context_ = self.app_context.clone(); + bus.set_sync_handler(move |_, msg| { + match msg.view() { + gstreamer::MessageView::NeedContext(ctxt) => { + if let Some(el) = msg + .src() + .map(|s| s.clone().downcast::().unwrap()) + { + let context_type = ctxt.context_type(); + if context_type == *gstreamer_gl::GL_DISPLAY_CONTEXT_TYPE { + let ctxt = gstreamer::Context::new(context_type, true); + ctxt.set_gl_display(&display_); + el.set_context(&ctxt); + } else if context_type == "gst.gl.app_context" { + let mut ctxt = gstreamer::Context::new(context_type, true); + { + let s = ctxt.get_mut().unwrap().structure_mut(); + s.set_value("context", context_.to_send_value()); + } + el.set_context(&ctxt); + } + } + }, + _ => (), + } + + gstreamer::BusSyncReply::Pass + }); + + let mut iter = vsinkbin + .dynamic_cast::() + .unwrap() + .iterate_elements(); + *self.gl_upload.lock().unwrap() = loop { + match iter.next() { + Ok(Some(element)) => { + if Some(true) == element.factory().map(|f| f.name() == "glupload") { + break Some(element); + } + }, + Err(gstreamer::IteratorError::Resync) => iter.resync(), + _ => break None, + } + }; + + Ok(()) + } +} diff --git a/components/media/backends/gstreamer/render-unix/Cargo.toml b/components/media/backends/gstreamer/render-unix/Cargo.toml new file mode 100644 index 00000000000..c61ee8abdf4 --- /dev/null +++ b/components/media/backends/gstreamer/render-unix/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "servo-media-gstreamer-render-unix" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[features] +gl-egl = ["gstreamer-gl-egl"] +gl-x11 = ["gstreamer-gl-x11"] +gl-wayland = ["gstreamer-gl-wayland"] + +[lib] +name = "servo_media_gstreamer_render_unix" +path = "lib.rs" + +[dependencies] +glib = { workspace = true } +gstreamer = { workspace = true } +gstreamer-gl = { workspace = true } +gstreamer-gl-egl = { workspace = true, optional = true } +gstreamer-gl-x11 = { workspace = true, optional = true } +gstreamer-gl-wayland = { workspace = true, optional = true } +gstreamer-video = { workspace = true } +sm-player = { package = "servo-media-player", path = "../../../player" } +sm-gst-render = { package = "servo-media-gstreamer-render", path = "../render" } diff --git a/components/media/backends/gstreamer/render-unix/lib.rs b/components/media/backends/gstreamer/render-unix/lib.rs new file mode 100644 index 00000000000..7300915ef24 --- /dev/null +++ b/components/media/backends/gstreamer/render-unix/lib.rs @@ -0,0 +1,288 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +//! `RenderUnix` is a `Render` implementation for Unix-based +//! platforms. It implements an OpenGL mechanism shared by Linux and +//! many of the BSD flavors. +//! +//! Internally it uses GStreamer's *glsinkbin* element as *videosink* +//! wrapping the *appsink* from the Player. And the shared frames are +//! mapped as texture IDs. + +use std::sync::{Arc, Mutex}; + +use gstreamer_gl::prelude::*; +use sm_gst_render::Render; +use sm_player::PlayerError; +use sm_player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; +use sm_player::video::{Buffer, VideoFrame, VideoFrameData}; + +struct GStreamerBuffer { + is_external_oes: bool, + frame: gstreamer_gl::GLVideoFrame, +} + +impl Buffer for GStreamerBuffer { + fn to_vec(&self) -> Option { + // packed formats are guaranteed to be in a single plane + if self.frame.format() == gstreamer_video::VideoFormat::Rgba { + let tex_id = self.frame.texture_id(0).ok()?; + Some(if self.is_external_oes { + VideoFrameData::OESTexture(tex_id) + } else { + VideoFrameData::Texture(tex_id) + }) + } else { + None + } + } +} + +pub struct RenderUnix { + display: gstreamer_gl::GLDisplay, + app_context: gstreamer_gl::GLContext, + gst_context: Arc>>, + gl_upload: Arc>>, +} + +impl RenderUnix { + /// Tries to create a new intance of the `RenderUnix` + /// + /// # Arguments + /// + /// * `context` - is the PlayerContext trait object from application. + pub fn new(app_gl_context: Box) -> Option { + // Check that we actually have the elements that we + // need to make this work. + gstreamer::ElementFactory::find("glsinkbin")?; + + let display_native = app_gl_context.get_native_display(); + let gl_context = app_gl_context.get_gl_context(); + let gl_api = match app_gl_context.get_gl_api() { + GlApi::OpenGL => gstreamer_gl::GLAPI::OPENGL, + GlApi::OpenGL3 => gstreamer_gl::GLAPI::OPENGL3, + GlApi::Gles1 => gstreamer_gl::GLAPI::GLES1, + GlApi::Gles2 => gstreamer_gl::GLAPI::GLES2, + GlApi::None => return None, + }; + + let (wrapped_context, display) = match gl_context { + GlContext::Egl(context) => { + let display = match display_native { + #[cfg(feature = "gl-egl")] + NativeDisplay::Egl(display_native) => { + unsafe { gstreamer_gl_egl::GLDisplayEGL::with_egl_display(display_native) } + .map(|display| display.upcast()) + .ok() + }, + #[cfg(feature = "gl-wayland")] + NativeDisplay::Wayland(display_native) => unsafe { + gstreamer_gl_wayland::GLDisplayWayland::with_display(display_native) + } + .map(|display| display.upcast()) + .ok(), + _ => None, + }; + + RenderUnix::create_wrapped_context( + display, + context, + gstreamer_gl::GLPlatform::EGL, + gl_api, + ) + }, + GlContext::Glx(context) => { + let display = match display_native { + #[cfg(feature = "gl-x11")] + NativeDisplay::X11(display_native) => { + unsafe { gstreamer_gl_x11::GLDisplayX11::with_display(display_native) } + .map(|display| display.upcast()) + .ok() + }, + _ => None, + }; + + RenderUnix::create_wrapped_context( + display, + context, + gstreamer_gl::GLPlatform::GLX, + gl_api, + ) + }, + GlContext::Unknown => (None, None), + }; + + match wrapped_context { + Some(app_context) => { + let cat = gstreamer::DebugCategory::get("servoplayer").unwrap(); + let _: Result<(), ()> = app_context + .activate(true) + .and_then(|_| { + app_context.fill_info().or_else(|err| { + gstreamer::warning!( + cat, + "Couldn't fill the wrapped app GL context: {}", + err.to_string() + ); + Ok(()) + }) + }) + .or_else(|_| { + gstreamer::warning!(cat, "Couldn't activate the wrapped app GL context"); + Ok(()) + }); + Some(RenderUnix { + display: display.unwrap(), + app_context, + gst_context: Arc::new(Mutex::new(None)), + gl_upload: Arc::new(Mutex::new(None)), + }) + }, + _ => None, + } + } + + fn create_wrapped_context( + display: Option, + handle: usize, + platform: gstreamer_gl::GLPlatform, + api: gstreamer_gl::GLAPI, + ) -> ( + Option, + Option, + ) { + match display { + Some(display) => { + let wrapped_context = unsafe { + gstreamer_gl::GLContext::new_wrapped(&display, handle, platform, api) + }; + (wrapped_context, Some(display)) + }, + _ => (None, None), + } + } +} + +impl Render for RenderUnix { + fn is_gl(&self) -> bool { + true + } + + fn build_frame(&self, sample: gstreamer::Sample) -> Option { + if self.gst_context.lock().unwrap().is_none() && self.gl_upload.lock().unwrap().is_some() { + *self.gst_context.lock().unwrap() = self + .gl_upload + .lock() + .unwrap() + .as_ref() + .map(|glupload| glupload.property::("context")); + } + + let buffer = sample.buffer_owned()?; + let caps = sample.caps()?; + + let is_external_oes = caps + .structure(0) + .and_then(|s| { + s.get::<&str>("texture-target").ok().and_then(|target| { + if target == "external-oes" { + Some(s) + } else { + None + } + }) + }) + .is_some(); + + let info = gstreamer_video::VideoInfo::from_caps(caps).ok()?; + let frame = gstreamer_gl::GLVideoFrame::from_buffer_readable(buffer, &info).ok()?; + VideoFrame::new( + info.width() as i32, + info.height() as i32, + Arc::new(GStreamerBuffer { + is_external_oes, + frame, + }), + ) + } + + fn build_video_sink( + &self, + appsink: &gstreamer::Element, + pipeline: &gstreamer::Element, + ) -> Result<(), PlayerError> { + if self.gl_upload.lock().unwrap().is_some() { + return Err(PlayerError::Backend( + "render unix already setup the video sink".to_owned(), + )); + } + + let vsinkbin = gstreamer::ElementFactory::make("glsinkbin") + .name("servo-media-vsink") + .build() + .map_err(|error| { + PlayerError::Backend(format!("glupload creation failed: {error:?}")) + })?; + + let caps = gstreamer::Caps::builder("video/x-raw") + .features([gstreamer_gl::CAPS_FEATURE_MEMORY_GL_MEMORY]) + .field("format", gstreamer_video::VideoFormat::Rgba.to_str()) + .field( + "texture-target", + gstreamer::List::new(["2D", "external-oes"]), + ) + .build(); + appsink.set_property("caps", caps); + + vsinkbin.set_property("sink", appsink); + + pipeline.set_property("video-sink", &vsinkbin); + + let bus = pipeline.bus().expect("pipeline with no bus"); + let display_ = self.display.clone(); + let context_ = self.app_context.clone(); + bus.set_sync_handler(move |_, msg| { + if let gstreamer::MessageView::NeedContext(ctxt) = msg.view() { + if let Some(el) = msg + .src() + .map(|s| s.clone().downcast::().unwrap()) + { + let context_type = ctxt.context_type(); + if context_type == *gstreamer_gl::GL_DISPLAY_CONTEXT_TYPE { + let ctxt = gstreamer::Context::new(context_type, true); + ctxt.set_gl_display(&display_); + el.set_context(&ctxt); + } else if context_type == "gst.gl.app_context" { + let mut ctxt = gstreamer::Context::new(context_type, true); + { + let s = ctxt.get_mut().unwrap().structure_mut(); + s.set_value("context", context_.to_send_value()); + } + el.set_context(&ctxt); + } + } + } + + gstreamer::BusSyncReply::Pass + }); + + let mut iter = vsinkbin + .dynamic_cast::() + .unwrap() + .iterate_elements(); + *self.gl_upload.lock().unwrap() = loop { + match iter.next() { + Ok(Some(element)) => { + if "glupload" == element.factory().unwrap().name() { + break Some(element); + } + }, + Err(gstreamer::IteratorError::Resync) => iter.resync(), + _ => break None, + } + }; + + Ok(()) + } +} diff --git a/components/media/backends/gstreamer/render.rs b/components/media/backends/gstreamer/render.rs new file mode 100644 index 00000000000..c8fe10d65c2 --- /dev/null +++ b/components/media/backends/gstreamer/render.rs @@ -0,0 +1,154 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::Arc; + +use glib::prelude::*; +use servo_media_gstreamer_render::Render; +use servo_media_player::PlayerError; +use servo_media_player::context::PlayerGLContext; +use servo_media_player::video::{Buffer, VideoFrame, VideoFrameData}; + +#[cfg(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" +))] +mod platform { + extern crate servo_media_gstreamer_render_unix; + pub use self::servo_media_gstreamer_render_unix::RenderUnix as Render; + use super::*; + + pub fn create_render(gl_context: Box) -> Option { + Render::new(gl_context) + } +} + +#[cfg(target_os = "android")] +mod platform { + extern crate servo_media_gstreamer_render_android; + pub use self::servo_media_gstreamer_render_android::RenderAndroid as Render; + use super::*; + + pub fn create_render(gl_context: Box) -> Option { + Render::new(gl_context) + } +} + +#[cfg(not(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "android", +)))] +mod platform { + use servo_media_gstreamer_render::Render as RenderTrait; + use servo_media_player::PlayerError; + use servo_media_player::context::PlayerGLContext; + use servo_media_player::video::VideoFrame; + + pub struct RenderDummy(); + pub type Render = RenderDummy; + + pub fn create_render(_: Box) -> Option { + None + } + + impl RenderTrait for RenderDummy { + fn is_gl(&self) -> bool { + false + } + + fn build_frame(&self, _: gstreamer::Sample) -> Option { + None + } + + fn build_video_sink( + &self, + _: &gstreamer::Element, + _: &gstreamer::Element, + ) -> Result<(), PlayerError> { + Err(PlayerError::Backend( + "Not available videosink decorator".to_owned(), + )) + } + } +} + +struct GStreamerBuffer { + frame: gstreamer_video::VideoFrame, +} + +impl Buffer for GStreamerBuffer { + fn to_vec(&self) -> Option { + let data = self.frame.plane_data(0).ok()?; + Some(VideoFrameData::Raw(Arc::new(data.to_vec()))) + } +} + +pub struct GStreamerRender { + render: Option, +} + +impl GStreamerRender { + pub fn new(gl_context: Box) -> Self { + GStreamerRender { + render: platform::create_render(gl_context), + } + } + + pub fn is_gl(&self) -> bool { + if let Some(render) = self.render.as_ref() { + render.is_gl() + } else { + false + } + } + + pub fn get_frame_from_sample(&self, sample: gstreamer::Sample) -> Option { + if let Some(render) = self.render.as_ref() { + render.build_frame(sample) + } else { + let buffer = sample.buffer_owned()?; + let caps = sample.caps()?; + let info = gstreamer_video::VideoInfo::from_caps(caps).ok()?; + let frame = gstreamer_video::VideoFrame::from_buffer_readable(buffer, &info).ok()?; + + VideoFrame::new( + info.width() as i32, + info.height() as i32, + Arc::new(GStreamerBuffer { frame }), + ) + } + } + + pub fn setup_video_sink( + &self, + pipeline: &gstreamer::Element, + ) -> Result { + let appsink = gstreamer::ElementFactory::make("appsink") + .build() + .map_err(|error| PlayerError::Backend(format!("appsink creation failed: {error:?}")))? + .downcast::() + .unwrap(); + + if let Some(render) = self.render.as_ref() { + render.build_video_sink(appsink.upcast_ref::(), pipeline)? + } else { + let caps = gstreamer::Caps::builder("video/x-raw") + .field("format", gstreamer_video::VideoFormat::Bgra.to_str()) + .field("pixel-aspect-ratio", gstreamer::Fraction::from((1, 1))) + .build(); + + appsink.set_caps(Some(&caps)); + pipeline.set_property("video-sink", &appsink); + }; + + Ok(appsink) + } +} diff --git a/components/media/backends/gstreamer/render/Cargo.toml b/components/media/backends/gstreamer/render/Cargo.toml new file mode 100644 index 00000000000..cb694c0ea2f --- /dev/null +++ b/components/media/backends/gstreamer/render/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "servo-media-gstreamer-render" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_gstreamer_render" +path = "lib.rs" + +[dependencies] +gstreamer = { workspace = true } +gstreamer-video = { workspace = true } + +[dependencies.sm-player] +package = "servo-media-player" +path = "../../../player" diff --git a/components/media/backends/gstreamer/render/lib.rs b/components/media/backends/gstreamer/render/lib.rs new file mode 100644 index 00000000000..7dce057b479 --- /dev/null +++ b/components/media/backends/gstreamer/render/lib.rs @@ -0,0 +1,49 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +//! `Render` is a trait to be used by GStreamer's backend player +//! +//! The purpose of this trait is to provide different accelerated +//! video renders. +//! +//! By default, the player will use a rendering mechanism based on +//! mapping the raw video into CPU memory, but it might be other +//! rendering mechanism. The main target for this trait are +//! OpenGL-based render mechanisms. +//! +//! Each platform (Unix, MacOS, Windows) might offer an implementation +//! of this trait, so the player could setup a proper GStreamer +//! pipeline, and handle the produced buffers. +//! + +pub trait Render { + /// Returns `True` if the render implementation uses any version + /// or flavor of OpenGL + fn is_gl(&self) -> bool; + + /// Returns the Player's `Frame` to be consumed by the API user. + /// + /// The implementation of this method will map the `sample`'s + /// buffer to the rendering appropriate structure. In the case of + /// OpenGL-based renders, the `Frame`, instead of the raw data, + /// will transfer the texture ID. + /// + /// # Arguments + /// + /// * `sample` - the GStreamer sample with the buffer to map + fn build_frame(&self, sample: gstreamer::Sample) -> Option; + + /// Sets the proper *video-sink* to GStreamer's `pipeline`, this + /// video sink is simply a decorator of the passed `appsink`. + /// + /// # Arguments + /// + /// * `appsink` - the appsink GStreamer element to decorate + /// * `pipeline` - the GStreamer pipeline to set the video sink + fn build_video_sink( + &self, + appsink: &gstreamer::Element, + pipeline: &gstreamer::Element, + ) -> Result<(), sm_player::PlayerError>; +} diff --git a/components/media/backends/gstreamer/source.rs b/components/media/backends/gstreamer/source.rs new file mode 100644 index 00000000000..d9b994a9aab --- /dev/null +++ b/components/media/backends/gstreamer/source.rs @@ -0,0 +1,426 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::convert::TryFrom; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use glib::subclass::prelude::*; +use gstreamer::prelude::*; +use gstreamer::subclass::prelude::*; +use once_cell::sync::Lazy; +use url::Url; + +const MAX_SRC_QUEUE_SIZE: u64 = 50 * 1024 * 1024; // 50 MB. + +// Implementation sub-module of the GObject +mod imp { + use super::*; + + macro_rules! inner_appsrc_proxy { + ($fn_name:ident, $return_type:ty) => { + pub fn $fn_name(&self) -> $return_type { + self.appsrc.$fn_name() + } + }; + + ($fn_name:ident, $arg1:ident, $arg1_type:ty, $return_type:ty) => { + pub fn $fn_name(&self, $arg1: $arg1_type) -> $return_type { + self.appsrc.$fn_name($arg1) + } + }; + } + + #[derive(Debug, Default)] + struct Position { + offset: u64, + requested_offset: u64, + } + + // The actual data structure that stores our values. This is not accessible + // directly from the outside. + pub struct ServoSrc { + cat: gstreamer::DebugCategory, + appsrc: gstreamer_app::AppSrc, + srcpad: gstreamer::GhostPad, + position: Mutex, + seeking: AtomicBool, + size: Mutex>, + } + + impl ServoSrc { + pub fn set_size(&self, size: i64) { + if self.seeking.load(Ordering::Relaxed) { + // We ignore set_size requests if we are seeking. + // The size value is temporarily stored so it + // is properly set once we are done seeking. + *self.size.lock().unwrap() = Some(size); + return; + } + + if self.appsrc.size() == -1 { + self.appsrc.set_size(size); + } + } + + pub fn set_seek_offset>(&self, parent: &O, offset: u64) -> bool { + let mut pos = self.position.lock().unwrap(); + + if pos.offset == offset || pos.requested_offset != 0 { + false + } else { + self.seeking.store(true, Ordering::Relaxed); + pos.requested_offset = offset; + gstreamer::debug!( + self.cat, + obj = parent, + "seeking to offset: {}", + pos.requested_offset + ); + + true + } + } + + pub fn set_seek_done(&self) { + self.seeking.store(false, Ordering::Relaxed); + + if let Some(size) = self.size.lock().unwrap().take() { + if self.appsrc.size() == -1 { + self.appsrc.set_size(size); + } + } + + let mut pos = self.position.lock().unwrap(); + pos.offset = pos.requested_offset; + pos.requested_offset = 0; + } + + pub fn push_buffer>( + &self, + parent: &O, + data: Vec, + ) -> Result { + if self.seeking.load(Ordering::Relaxed) { + gstreamer::debug!(self.cat, obj = parent, "seek in progress, ignored data"); + return Ok(gstreamer::FlowSuccess::Ok); + } + + let mut pos = self.position.lock().unwrap(); // will block seeking + + let length = u64::try_from(data.len()).unwrap(); + let mut data_offset = 0; + + let buffer_starting_offset = pos.offset; + + // @TODO: optimization: update the element's blocksize by + // X factor given current length + + pos.offset += length; + + gstreamer::trace!(self.cat, obj = parent, "offset: {}", pos.offset); + + // set the stream size (in bytes) to current offset if + // size is lesser than it + if let Ok(size) = u64::try_from(self.appsrc.size()) { + if pos.offset > size { + gstreamer::debug!( + self.cat, + obj = parent, + "Updating internal size from {} to {}", + size, + pos.offset + ); + let new_size = i64::try_from(pos.offset).unwrap(); + self.appsrc.set_size(new_size); + } + } + + // Split the received vec<> into buffers that are of a + // size basesrc suggest. It is important not to push + // buffers that are too large, otherwise incorrect + // buffering messages can be sent from the pipeline + let block_size = 4096; + let num_blocks = ((length - data_offset) as f64 / block_size as f64).ceil() as u64; + + gstreamer::log!( + self.cat, + obj = parent, + "Splitting the received vec into {} blocks", + num_blocks + ); + + let mut ret: Result = + Ok(gstreamer::FlowSuccess::Ok); + for i in 0..num_blocks { + let start = usize::try_from(i * block_size + data_offset).unwrap(); + data_offset = 0; + let size = usize::try_from(block_size.min(length - start as u64)).unwrap(); + let end = start + size; + + let buffer_offset = buffer_starting_offset + start as u64; + let buffer_offset_end = buffer_offset + size as u64; + + let subdata = Vec::from(&data[start..end]); + let mut buffer = gstreamer::Buffer::from_slice(subdata); + { + let buffer = buffer.get_mut().unwrap(); + buffer.set_offset(buffer_offset); + buffer.set_offset_end(buffer_offset_end); + } + + if self.seeking.load(Ordering::Relaxed) { + gstreamer::trace!( + self.cat, + obj = parent, + "stopping buffer appends due to seek" + ); + ret = Ok(gstreamer::FlowSuccess::Ok); + break; + } + + gstreamer::trace!(self.cat, obj = parent, "Pushing buffer {:?}", buffer); + + ret = self.appsrc.push_buffer(buffer); + match ret { + Ok(_) => (), + Err(gstreamer::FlowError::Eos) | Err(gstreamer::FlowError::Flushing) => { + ret = Ok(gstreamer::FlowSuccess::Ok) + }, + Err(_) => break, + } + } + + ret + } + + inner_appsrc_proxy!(end_of_stream, Result); + inner_appsrc_proxy!(set_callbacks, callbacks, gstreamer_app::AppSrcCallbacks, ()); + + fn query(&self, pad: &gstreamer::GhostPad, query: &mut gstreamer::QueryRef) -> bool { + gstreamer::log!(self.cat, obj = pad, "Handling query {:?}", query); + + // In order to make buffering/downloading work as we want, apart from + // setting the appropriate flags on the player playbin, + // the source needs to either: + // + // 1. be an http, mms, etc. scheme + // 2. report that it is "bandwidth limited". + // + // 1. is not straightforward because we are using a servosrc scheme for now. + // This may change in the future if we end up handling http/https/data + // URIs, which is what WebKit does. + // + // For 2. we need to make servosrc handle the scheduling properties query + // to report that it "is bandwidth limited". + let ret = match query.view_mut() { + gstreamer::QueryViewMut::Scheduling(ref mut q) => { + let flags = gstreamer::SchedulingFlags::SEQUENTIAL | + gstreamer::SchedulingFlags::BANDWIDTH_LIMITED; + q.set(flags, 1, -1, 0); + q.add_scheduling_modes([gstreamer::PadMode::Push]); + true + }, + _ => gstreamer::Pad::query_default(pad, Some(&*self.obj()), query), + }; + + if ret { + gstreamer::log!(self.cat, obj = pad, "Handled query {:?}", query); + } else { + gstreamer::info!(self.cat, obj = pad, "Didn't handle query {:?}", query); + } + ret + } + } + + // Basic declaration of our type for the GObject type system + #[glib::object_subclass] + impl ObjectSubclass for ServoSrc { + const NAME: &'static str = "ServoSrc"; + type Type = super::ServoSrc; + type ParentType = gstreamer::Bin; + type Interfaces = (gstreamer::URIHandler,); + + // Called once at the very beginning of instantiation of each instance and + // creates the data structure that contains all our state + fn with_class(klass: &Self::Class) -> Self { + let app_src = gstreamer::ElementFactory::make("appsrc") + .build() + .map(|elem| elem.downcast::().unwrap()) + .expect("Could not create appsrc element"); + + let pad_templ = klass.pad_template("src").unwrap(); + let ghost_pad = gstreamer::GhostPad::builder_from_template(&pad_templ) + .query_function(|pad, parent, query| { + ServoSrc::catch_panic_pad_function( + parent, + || false, + |servosrc| servosrc.query(pad, query), + ) + }) + .build(); + + Self { + cat: gstreamer::DebugCategory::new( + "servosrc", + gstreamer::DebugColorFlags::empty(), + Some("Servo source"), + ), + appsrc: app_src, + srcpad: ghost_pad, + position: Mutex::new(Default::default()), + seeking: AtomicBool::new(false), + size: Mutex::new(None), + } + } + } + + // The ObjectImpl trait provides the setters/getters for GObject properties. + // Here we need to provide the values that are internally stored back to the + // caller, or store whatever new value the caller is providing. + // + // This maps between the GObject properties and our internal storage of the + // corresponding values of the properties. + impl ObjectImpl for ServoSrc { + // Called right after construction of a new instance + fn constructed(&self) { + // Call the parent class' ::constructed() implementation first + self.parent_constructed(); + + self.obj() + .add(&self.appsrc) + .expect("Could not add appsrc element to bin"); + + let target_pad = self.appsrc.static_pad("src"); + self.srcpad.set_target(target_pad.as_ref()).unwrap(); + + self.obj() + .add_pad(&self.srcpad) + .expect("Could not add source pad to bin"); + + self.appsrc.set_caps(None::<&gstreamer::Caps>); + self.appsrc.set_max_bytes(MAX_SRC_QUEUE_SIZE); + self.appsrc.set_block(false); + self.appsrc.set_format(gstreamer::Format::Bytes); + self.appsrc + .set_stream_type(gstreamer_app::AppStreamType::Seekable); + + self.obj() + .set_element_flags(gstreamer::ElementFlags::SOURCE); + } + } + + impl GstObjectImpl for ServoSrc {} + + // Implementation of gstreamer::Element virtual methods + impl ElementImpl for ServoSrc { + fn metadata() -> Option<&'static gstreamer::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gstreamer::subclass::ElementMetadata::new( + "Servo Media Source", + "Source/Audio/Video", + "Feed player with media data", + "Servo developers", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gstreamer::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let caps = gstreamer::Caps::new_any(); + let src_pad_template = gstreamer::PadTemplate::new( + "src", + gstreamer::PadDirection::Src, + gstreamer::PadPresence::Always, + &caps, + ) + .unwrap(); + + vec![src_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } + } + + // Implementation of gstreamer::Bin virtual methods + impl BinImpl for ServoSrc {} + + impl URIHandlerImpl for ServoSrc { + const URI_TYPE: gstreamer::URIType = gstreamer::URIType::Src; + + fn protocols() -> &'static [&'static str] { + &["servosrc"] + } + + fn uri(&self) -> Option { + Some("servosrc://".to_string()) + } + + fn set_uri(&self, uri: &str) -> Result<(), glib::Error> { + if let Ok(uri) = Url::parse(uri) { + if uri.scheme() == "servosrc" { + return Ok(()); + } + } + Err(glib::Error::new( + gstreamer::URIError::BadUri, + format!("Invalid URI '{:?}'", uri,).as_str(), + )) + } + } +} + +// Public part of the ServoSrc type. This behaves like a normal +// GObject binding +glib::wrapper! { + pub struct ServoSrc(ObjectSubclass) + @extends gstreamer::Bin, gstreamer::Element, gstreamer::Object, @implements gstreamer::URIHandler; +} + +unsafe impl Send for ServoSrc {} +unsafe impl Sync for ServoSrc {} + +impl ServoSrc { + pub fn set_size(&self, size: i64) { + self.imp().set_size(size); + } + + pub fn set_seek_offset(&self, offset: u64) -> bool { + self.imp().set_seek_offset(self, offset) + } + + pub fn set_seek_done(&self) { + self.imp().set_seek_done(); + } + + pub fn push_buffer( + &self, + data: Vec, + ) -> Result { + self.imp().push_buffer(self, data) + } + + pub fn push_end_of_stream(&self) -> Result { + self.imp().end_of_stream() + } + + pub fn set_callbacks(&self, callbacks: gstreamer_app::AppSrcCallbacks) { + self.imp().set_callbacks(callbacks) + } +} + +// Registers the type for our element, and then registers in GStreamer +// under the name "servosrc" for being able to instantiate it via e.g. +// gstreamer::ElementFactory::make(). +pub fn register_servo_src() -> Result<(), glib::BoolError> { + gstreamer::Element::register( + None, + "servosrc", + gstreamer::Rank::NONE, + ServoSrc::static_type(), + ) +} diff --git a/components/media/backends/gstreamer/webrtc.rs b/components/media/backends/gstreamer/webrtc.rs new file mode 100644 index 00000000000..781dc626278 --- /dev/null +++ b/components/media/backends/gstreamer/webrtc.rs @@ -0,0 +1,815 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::{cmp, mem}; + +use glib::prelude::*; +use gstreamer::prelude::*; +use log::warn; +use servo_media_streams::MediaStreamType; +use servo_media_streams::registry::{MediaStreamId, get_stream}; +use servo_media_webrtc::datachannel::DataChannelId; +use servo_media_webrtc::thread::InternalEvent; +use servo_media_webrtc::{WebRtcController as WebRtcThread, *}; +use {glib, gstreamer, gstreamer_sdp, gstreamer_webrtc}; + +use super::BACKEND_BASE_TIME; +use crate::datachannel::GStreamerWebRtcDataChannel; +use crate::media_stream::GStreamerMediaStream; + +// TODO: +// - figure out purpose of glib loop + +#[derive(Debug, Clone)] +pub struct MLineInfo { + /// The caps for the given m-line + caps: gstreamer::Caps, + /// Whether or not this sink pad has already been connected + is_used: bool, + /// The payload value of the given m-line + payload: i32, +} + +enum DataChannelEventTarget { + Buffered(Vec), + Created(GStreamerWebRtcDataChannel), +} + +pub struct GStreamerWebRtcController { + webrtc: gstreamer::Element, + pipeline: gstreamer::Pipeline, + /// We can't trigger a negotiation-needed event until we have streams, or otherwise + /// a createOffer() call will lead to bad SDP. Instead, we delay negotiation. + delayed_negotiation: bool, + /// A handle to the event loop abstraction surrounding the webrtc implementations, + /// which lets gstreamer callbacks send events back to the event loop to run on this object + thread: WebRtcThread, + signaller: Box, + /// All the streams that are actually connected to the webrtcbin (i.e., their presence has already + /// been negotiated) + streams: Vec, + /// Disconnected streams that are waiting to be linked. Streams are + /// only linked when: + /// + /// - An offer is made (all pending streams are flushed) + /// - An offer is received (all matching pending streams are flushed) + /// - A stream is added when there is a so-far-disconnected remote-m-line + /// + /// In other words, these are all yet to be negotiated + /// + /// See link_stream + pending_streams: Vec, + /// Each new webrtc stream should have a new payload/pt value, starting at 96 + /// + /// This is maintained as a known yet-unused payload number, being incremented whenever + /// we use it, and set to (remote_pt + 1) if the remote sends us a stream with a higher pt + pt_counter: i32, + /// We keep track of how many request pads have been created on webrtcbin + /// so that we can request more to fill in the gaps and acquire a specific pad if necessary + request_pad_counter: usize, + /// Streams need to be connected to the relevant sink pad, and we figure this out + /// by keeping track of the caps of each m-line in the SDP. + remote_mline_info: Vec, + /// Temporary storage for remote_mline_info until the remote description is applied + /// + /// Without this, a unluckily timed call to link_stream() may happen before the webrtcbin + /// knows the remote description, but while we _think_ it does + pending_remote_mline_info: Vec, + /// In case we get multiple remote offers, this lets us keep track of which is the newest + remote_offer_generation: u32, + _main_loop: glib::MainLoop, + data_channels: Arc>>, + next_data_channel_id: Arc, +} + +impl WebRtcControllerBackend for GStreamerWebRtcController { + fn add_ice_candidate(&mut self, candidate: IceCandidate) -> WebRtcResult { + self.webrtc.emit_by_name::<()>( + "add-ice-candidate", + &[&candidate.sdp_mline_index, &candidate.candidate], + ); + Ok(()) + } + + fn set_remote_description( + &mut self, + desc: SessionDescription, + cb: Box, + ) -> WebRtcResult { + self.set_description(desc, DescriptionType::Remote, cb) + } + + fn set_local_description( + &mut self, + desc: SessionDescription, + cb: Box, + ) -> WebRtcResult { + self.set_description(desc, DescriptionType::Local, cb) + } + + fn create_offer( + &mut self, + cb: Box, + ) -> WebRtcResult { + self.flush_pending_streams(true)?; + self.pipeline.set_state(gstreamer::State::Playing)?; + let promise = gstreamer::Promise::with_change_func(move |res| { + res.map(|s| on_offer_or_answer_created(SdpType::Offer, s.unwrap(), cb)) + .unwrap(); + }); + + self.webrtc + .emit_by_name::<()>("create-offer", &[&None::, &promise]); + Ok(()) + } + + fn create_answer( + &mut self, + cb: Box, + ) -> WebRtcResult { + let promise = gstreamer::Promise::with_change_func(move |res| { + res.map(|s| on_offer_or_answer_created(SdpType::Answer, s.unwrap(), cb)) + .unwrap(); + }); + + self.webrtc + .emit_by_name::<()>("create-answer", &[&None::, &promise]); + Ok(()) + } + + fn add_stream(&mut self, stream_id: &MediaStreamId) -> WebRtcResult { + let stream = + get_stream(stream_id).expect("Media streams registry does not contain such ID"); + let mut stream = stream.lock().unwrap(); + let stream = stream + .as_mut_any() + .downcast_mut::() + .ok_or("Does not currently support non-gstreamer streams")?; + self.link_stream(stream_id, stream, false)?; + if self.delayed_negotiation && (self.streams.len() > 1 || self.pending_streams.len() > 1) { + self.delayed_negotiation = false; + self.signaller.on_negotiation_needed(&self.thread); + } + Ok(()) + } + + fn create_data_channel(&mut self, init: &DataChannelInit) -> WebRtcDataChannelResult { + let id = self.next_data_channel_id.fetch_add(1, Ordering::Relaxed); + match GStreamerWebRtcDataChannel::new(&id, &self.webrtc, &self.thread, init) { + Ok(channel) => register_data_channel(self.data_channels.clone(), id, channel), + Err(error) => Err(WebRtcError::Backend(error)), + } + } + + fn close_data_channel(&mut self, id: &DataChannelId) -> WebRtcResult { + // There is no need to unregister the channel here. It will be unregistered + // when the data channel backend triggers the on closed event. + let mut data_channels = self.data_channels.lock().unwrap(); + match data_channels.get(id) { + Some(ref channel) => match channel { + DataChannelEventTarget::Created(channel) => { + channel.close(); + Ok(()) + }, + DataChannelEventTarget::Buffered(_) => data_channels + .remove(id) + .ok_or(WebRtcError::Backend("Unknown data channel".to_owned())) + .map(|_| ()), + }, + None => Err(WebRtcError::Backend("Unknown data channel".to_owned())), + } + } + + fn send_data_channel_message( + &mut self, + id: &DataChannelId, + message: &DataChannelMessage, + ) -> WebRtcResult { + match self.data_channels.lock().unwrap().get(id) { + Some(ref channel) => match channel { + DataChannelEventTarget::Created(channel) => { + channel.send(message); + Ok(()) + }, + _ => Ok(()), + }, + None => Err(WebRtcError::Backend("Unknown data channel".to_owned())), + } + } + + fn configure(&mut self, stun_server: &str, policy: BundlePolicy) -> WebRtcResult { + self.webrtc + .set_property_from_str("stun-server", stun_server); + self.webrtc + .set_property_from_str("bundle-policy", policy.as_str()); + Ok(()) + } + + fn internal_event(&mut self, e: thread::InternalEvent) -> WebRtcResult { + match e { + InternalEvent::OnNegotiationNeeded => { + if self.streams.is_empty() && self.pending_streams.is_empty() { + // we have no streams + + // If the pipeline starts playing and on-negotiation-needed is present before there are any + // media streams, an invalid SDP offer will be created. Therefore, delay emitting the signal + self.delayed_negotiation = true; + } else { + self.signaller.on_negotiation_needed(&self.thread); + } + }, + InternalEvent::OnIceCandidate(candidate) => { + self.signaller.on_ice_candidate(&self.thread, candidate); + }, + InternalEvent::OnAddStream(stream, ty) => { + self.pipeline.set_state(gstreamer::State::Playing)?; + self.signaller.on_add_stream(&stream, ty); + }, + InternalEvent::OnDataChannelEvent(channel_id, event) => { + let mut data_channels = self.data_channels.lock().unwrap(); + match data_channels.get_mut(&channel_id) { + None => { + data_channels + .insert(channel_id, DataChannelEventTarget::Buffered(vec![event])); + }, + Some(ref mut channel) => match channel { + &mut &mut DataChannelEventTarget::Buffered(ref mut events) => { + events.push(event); + return Ok(()); + }, + DataChannelEventTarget::Created(_) => { + if let DataChannelEvent::Close = event { + data_channels.remove(&channel_id); + } + self.signaller + .on_data_channel_event(channel_id, event, &self.thread); + }, + }, + } + }, + InternalEvent::DescriptionAdded(cb, description_type, ty, remote_offer_generation) => { + if description_type == DescriptionType::Remote && + ty == SdpType::Offer && + remote_offer_generation == self.remote_offer_generation + { + mem::swap( + &mut self.pending_remote_mline_info, + &mut self.remote_mline_info, + ); + self.pending_remote_mline_info.clear(); + self.flush_pending_streams(false)?; + } + self.pipeline.set_state(gstreamer::State::Playing)?; + cb(); + }, + InternalEvent::UpdateSignalingState => { + use gstreamer_webrtc::WebRTCSignalingState::*; + let val = self + .webrtc + .property::("signaling-state"); + let state = match val { + Stable => SignalingState::Stable, + HaveLocalOffer => SignalingState::HaveLocalOffer, + HaveRemoteOffer => SignalingState::HaveRemoteOffer, + HaveLocalPranswer => SignalingState::HaveLocalPranswer, + HaveRemotePranswer => SignalingState::HaveRemotePranswer, + Closed => SignalingState::Closed, + i => { + return Err(WebRtcError::Backend(format!( + "unknown signaling state: {:?}", + i + ))); + }, + }; + self.signaller.update_signaling_state(state); + }, + InternalEvent::UpdateGatheringState => { + use gstreamer_webrtc::WebRTCICEGatheringState::*; + let val = self + .webrtc + .property::("ice-gathering-state"); + let state = match val { + New => GatheringState::New, + Gathering => GatheringState::Gathering, + Complete => GatheringState::Complete, + i => { + return Err(WebRtcError::Backend(format!( + "unknown gathering state: {:?}", + i + ))); + }, + }; + self.signaller.update_gathering_state(state); + }, + InternalEvent::UpdateIceConnectionState => { + use gstreamer_webrtc::WebRTCICEConnectionState::*; + let val = self + .webrtc + .property::("ice-connection-state"); + let state = match val { + New => IceConnectionState::New, + Checking => IceConnectionState::Checking, + Connected => IceConnectionState::Connected, + Completed => IceConnectionState::Completed, + Disconnected => IceConnectionState::Disconnected, + Failed => IceConnectionState::Failed, + Closed => IceConnectionState::Closed, + i => { + return Err(WebRtcError::Backend(format!( + "unknown ICE connection state: {:?}", + i + ))); + }, + }; + self.signaller.update_ice_connection_state(state); + }, + } + Ok(()) + } + + fn quit(&mut self) { + self.signaller.close(); + + self.pipeline.set_state(gstreamer::State::Null).unwrap(); + } +} + +impl GStreamerWebRtcController { + fn set_description( + &mut self, + desc: SessionDescription, + description_type: DescriptionType, + cb: Box, + ) -> WebRtcResult { + let ty = match desc.type_ { + SdpType::Answer => gstreamer_webrtc::WebRTCSDPType::Answer, + SdpType::Offer => gstreamer_webrtc::WebRTCSDPType::Offer, + SdpType::Pranswer => gstreamer_webrtc::WebRTCSDPType::Pranswer, + SdpType::Rollback => gstreamer_webrtc::WebRTCSDPType::Rollback, + }; + + let kind = match description_type { + DescriptionType::Local => "set-local-description", + DescriptionType::Remote => "set-remote-description", + }; + + let sdp = gstreamer_sdp::SDPMessage::parse_buffer(desc.sdp.as_bytes()).unwrap(); + if description_type == DescriptionType::Remote { + self.remote_offer_generation += 1; + self.store_remote_mline_info(&sdp); + } + let answer = gstreamer_webrtc::WebRTCSessionDescription::new(ty, sdp); + let thread = self.thread.clone(); + let remote_offer_generation = self.remote_offer_generation; + let promise = gstreamer::Promise::with_change_func(move |_promise| { + // remote_offer_generation here ensures that DescriptionAdded doesn't + // flush pending_remote_mline_info for stale remote offer callbacks + thread.internal_event(InternalEvent::DescriptionAdded( + cb, + description_type, + desc.type_, + remote_offer_generation, + )); + }); + self.webrtc.emit_by_name::<()>(kind, &[&answer, &promise]); + Ok(()) + } + + fn store_remote_mline_info(&mut self, sdp: &gstreamer_sdp::SDPMessage) { + self.pending_remote_mline_info.clear(); + for media in sdp.medias() { + let mut caps = gstreamer::Caps::new_empty(); + let caps_mut = caps.get_mut().expect("Fresh caps should be uniquely owned"); + for format in media.formats() { + if format == "webrtc-datachannel" { + return; + } + let pt = format + .parse() + .expect("Gstreamer provided noninteger format"); + caps_mut.append( + media + .caps_from_media(pt) + .expect("get_format() did not return a format from the SDP"), + ); + self.pt_counter = cmp::max(self.pt_counter, pt + 1); + } + for s in caps_mut.iter_mut() { + // the caps are application/x-unknown by default, which will fail + // to intersect + // + // see https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/blob/ba62917fbfd98ea76d4e066a6f18b4a14b847362/ext/webrtc/gstwebrtcbin.c#L2521 + s.set_name("application/x-rtp") + } + // This info is not current until the promise from set-remote-description is resolved, + // to avoid any races where we attempt to link streams before the promise resolves we + // queue this up in a pending buffer + self.pending_remote_mline_info.push(MLineInfo { + caps, + // XXXManishearth in the (yet unsupported) case of dynamic stream addition and renegotiation + // this will need to be checked against the current set of streams + is_used: false, + // XXXManishearth ideally, we keep track of all payloads and have the capability of picking + // the appropriate decoder. For this, a bunch of the streams code will have to be moved into + // a webrtc-specific abstraction. + payload: media + .format(0) + .expect("Gstreamer reported incorrect formats_len()") + .parse() + .expect("Gstreamer provided noninteger format"), + }); + } + } + + /// Streams need to be linked to the correct pads, so we buffer them up until we know enough + /// to do this. + /// + /// When we get a remote offer, we store the relevant m-line information so that we can + /// pick the correct sink pad and payload. Shortly after we look for any pending streams + /// and connect them to available compatible m-lines using link_stream. + /// + /// When we create an offer, we're controlling the pad order, so we set request_new_pads + /// to true and forcefully link all pending streams before generating the offer. + /// + /// When request_new_pads is false, we may still request new pads, however we only do this for + /// streams that have already been negotiated by the remote. + fn link_stream( + &mut self, + stream_id: &MediaStreamId, + stream: &mut GStreamerMediaStream, + request_new_pads: bool, + ) -> WebRtcResult { + let caps = stream.caps(); + let idx = self + .remote_mline_info + .iter() + .enumerate() + .filter(|(_, x)| !x.is_used) + .find(|(_, x)| x.caps.can_intersect(caps)) + .map(|x| x.0); + if let Some(idx) = idx { + if idx >= self.request_pad_counter { + for i in self.request_pad_counter..=idx { + // webrtcbin needs you to request pads (or use element.link(webrtcbin)) + // however, it also wants them to be connected in the correct order. + // + // Here, we make sure all the numbered sink pads have been created beforehand, up to + // and including the one we need here. + // + // An alternate fix is to sort pending_streams according to the m-line index + // and just do it in order. This also seems brittle. + self.webrtc + .request_pad_simple(&format!("sink_{}", i)) + .ok_or("Cannot request sink pad")?; + } + self.request_pad_counter = idx + 1; + } + stream.attach_to_pipeline(&self.pipeline); + let element = stream.encoded(); + self.remote_mline_info[idx].is_used = true; + let caps = stream.caps_with_payload(self.remote_mline_info[idx].payload); + element.set_property("caps", &caps); + let src = element.static_pad("src").ok_or("Cannot request src pad")?; + let sink = self + .webrtc + .static_pad(&format!("sink_{}", idx)) + .ok_or("Cannot request sink pad")?; + src.link(&sink)?; + self.streams.push(*stream_id); + } else if request_new_pads { + stream.attach_to_pipeline(&self.pipeline); + let element = stream.encoded(); + let caps = stream.caps_with_payload(self.pt_counter); + self.pt_counter += 1; + element.set_property("caps", &caps); + let src = element.static_pad("src").ok_or("Cannot request src pad")?; + let sink = self + .webrtc + .request_pad_simple(&format!("sink_{}", self.request_pad_counter)) + .ok_or("Cannot request sink pad")?; + self.request_pad_counter += 1; + src.link(&sink)?; + self.streams.push(*stream_id); + } else { + self.pending_streams.push(*stream_id); + } + Ok(()) + } + + /// link_stream, but for all pending streams + fn flush_pending_streams(&mut self, request_new_pads: bool) -> WebRtcResult { + let pending_streams = std::mem::take(&mut self.pending_streams); + for stream_id in pending_streams { + let stream = + get_stream(&stream_id).expect("Media streams registry does not contain such ID"); + let mut stream = stream.lock().unwrap(); + let stream = stream + .as_mut_any() + .downcast_mut::() + .ok_or("Does not currently support non-gstreamer streams")?; + self.link_stream(&stream_id, stream, request_new_pads)?; + } + Ok(()) + } + + fn start_pipeline(&mut self) -> WebRtcResult { + self.pipeline.add(&self.webrtc)?; + + // gstreamer needs Sync on these callbacks for some reason + // https://github.com/sdroege/gstreamer-rs/issues/154 + let thread = Mutex::new(self.thread.clone()); + self.webrtc + .connect("on-ice-candidate", false, move |values| { + thread + .lock() + .unwrap() + .internal_event(InternalEvent::OnIceCandidate(candidate(values))); + None + }); + + let thread = Arc::new(Mutex::new(self.thread.clone())); + self.webrtc.connect_pad_added({ + let pipeline_weak = self.pipeline.downgrade(); + move |_element, pad| { + let Some(pipe) = pipeline_weak.upgrade() else { + warn!("Pipeline already deallocated"); + return; + }; + process_new_stream(pad, &pipe, thread.clone()); + } + }); + + // gstreamer needs Sync on these callbacks for some reason + // https://github.com/sdroege/gstreamer-rs/issues/154 + let thread = Mutex::new(self.thread.clone()); + self.webrtc + .connect("on-negotiation-needed", false, move |_values| { + thread + .lock() + .unwrap() + .internal_event(InternalEvent::OnNegotiationNeeded); + None + }); + + let thread = Mutex::new(self.thread.clone()); + self.webrtc + .connect("notify::signaling-state", false, move |_values| { + thread + .lock() + .unwrap() + .internal_event(InternalEvent::UpdateSignalingState); + None + }); + let thread = Mutex::new(self.thread.clone()); + self.webrtc + .connect("notify::ice-connection-state", false, move |_values| { + thread + .lock() + .unwrap() + .internal_event(InternalEvent::UpdateIceConnectionState); + None + }); + let thread = Mutex::new(self.thread.clone()); + self.webrtc + .connect("notify::ice-gathering-state", false, move |_values| { + thread + .lock() + .unwrap() + .internal_event(InternalEvent::UpdateGatheringState); + None + }); + let thread = Mutex::new(self.thread.clone()); + let data_channels = self.data_channels.clone(); + let next_data_channel_id = self.next_data_channel_id.clone(); + self.webrtc + .connect("on-data-channel", false, move |channel| { + let channel = channel[1] + .get::() + .map_err(|e| e.to_string()) + .expect("Invalid data channel"); + let id = next_data_channel_id.fetch_add(1, Ordering::Relaxed); + let thread_ = thread.lock().unwrap().clone(); + match GStreamerWebRtcDataChannel::from(&id, channel, &thread_) { + Ok(channel) => { + let mut closed_channel = false; + { + thread_.internal_event(InternalEvent::OnDataChannelEvent( + id, + DataChannelEvent::NewChannel, + )); + + let mut data_channels = data_channels.lock().unwrap(); + if let Some(ref mut channel) = data_channels.get_mut(&id) { + match channel { + &mut &mut DataChannelEventTarget::Buffered(ref mut events) => { + for event in events.drain(0..) { + if let DataChannelEvent::Close = event { + closed_channel = true + } + thread_.internal_event( + InternalEvent::OnDataChannelEvent(id, event), + ); + } + }, + _ => debug_assert!( + false, + "Trying to register a data channel with an existing ID" + ), + } + } + data_channels.remove(&id); + } + if !closed_channel && + register_data_channel(data_channels.clone(), id, channel).is_err() + { + warn!("Could not register data channel {:?}", id); + return None; + } + }, + Err(error) => { + warn!("Could not create data channel {:?}", error); + }, + } + None + }); + + self.pipeline.set_state(gstreamer::State::Ready)?; + Ok(()) + } +} + +pub fn construct( + signaller: Box, + thread: WebRtcThread, +) -> Result { + let main_loop = glib::MainLoop::new(None, false); + let pipeline = gstreamer::Pipeline::with_name("webrtc main"); + pipeline.set_start_time(gstreamer::ClockTime::NONE); + pipeline.set_base_time(*BACKEND_BASE_TIME); + pipeline.use_clock(Some(&gstreamer::SystemClock::obtain())); + let webrtc = gstreamer::ElementFactory::make("webrtcbin") + .name("sendrecv") + .build() + .map_err(|error| format!("webrtcbin element not found: {error:?}"))?; + let mut controller = GStreamerWebRtcController { + webrtc, + pipeline, + signaller, + thread, + remote_mline_info: vec![], + pending_remote_mline_info: vec![], + streams: vec![], + pending_streams: vec![], + pt_counter: 96, + request_pad_counter: 0, + remote_offer_generation: 0, + delayed_negotiation: false, + _main_loop: main_loop, + data_channels: Arc::new(Mutex::new(HashMap::new())), + next_data_channel_id: Arc::new(AtomicUsize::new(0)), + }; + controller.start_pipeline()?; + Ok(controller) +} + +fn on_offer_or_answer_created( + ty: SdpType, + reply: &gstreamer::StructureRef, + cb: Box, +) { + debug_assert!(ty == SdpType::Offer || ty == SdpType::Answer); + let reply = reply + .value(ty.as_str()) + .unwrap() + .get::() + .expect("Invalid argument"); + + let type_ = match reply.type_() { + gstreamer_webrtc::WebRTCSDPType::Answer => SdpType::Answer, + gstreamer_webrtc::WebRTCSDPType::Offer => SdpType::Offer, + gstreamer_webrtc::WebRTCSDPType::Pranswer => SdpType::Pranswer, + gstreamer_webrtc::WebRTCSDPType::Rollback => SdpType::Rollback, + _ => panic!("unknown sdp response"), + }; + + let desc = SessionDescription { + sdp: reply.sdp().as_text().unwrap(), + type_, + }; + + cb(desc); +} + +fn on_incoming_stream( + pipe: &gstreamer::Pipeline, + thread: Arc>, + pad: &gstreamer::Pad, +) { + let decodebin = gstreamer::ElementFactory::make("decodebin") + .build() + .unwrap(); + let caps = pad.query_caps(None); + let name = caps + .structure(0) + .unwrap() + .get::("media") + .expect("Invalid 'media' field"); + let decodebin2 = decodebin.clone(); + decodebin.connect_pad_added({ + let pipeline_weak = pipe.downgrade(); + move |_element, pad| { + let Some(pipe) = pipeline_weak.upgrade() else { + warn!("Pipeline already deallocated"); + return; + }; + on_incoming_decodebin_stream(pad, &pipe, thread.clone(), &name); + } + }); + pipe.add(&decodebin).unwrap(); + + let decodepad = decodebin.static_pad("sink").unwrap(); + pad.link(&decodepad).unwrap(); + decodebin2.sync_state_with_parent().unwrap(); +} + +fn on_incoming_decodebin_stream( + pad: &gstreamer::Pad, + pipe: &gstreamer::Pipeline, + thread: Arc>, + name: &str, +) { + let proxy_sink = gstreamer::ElementFactory::make("proxysink") + .build() + .unwrap(); + let proxy_src = gstreamer::ElementFactory::make("proxysrc") + .property("proxysink", &proxy_sink) + .build() + .unwrap(); + pipe.add(&proxy_sink).unwrap(); + let sinkpad = proxy_sink.static_pad("sink").unwrap(); + + pad.link(&sinkpad).unwrap(); + proxy_sink.sync_state_with_parent().unwrap(); + + let (stream, ty) = if name == "video" { + ( + GStreamerMediaStream::create_video_from(proxy_src), + MediaStreamType::Video, + ) + } else { + ( + GStreamerMediaStream::create_audio_from(proxy_src), + MediaStreamType::Audio, + ) + }; + thread + .lock() + .unwrap() + .internal_event(InternalEvent::OnAddStream(stream, ty)); +} + +fn process_new_stream( + pad: &gstreamer::Pad, + pipe: &gstreamer::Pipeline, + thread: Arc>, +) { + if pad.direction() != gstreamer::PadDirection::Src { + // Ignore outgoing pad notifications. + return; + } + on_incoming_stream(pipe, thread, pad) +} + +fn candidate(values: &[glib::Value]) -> IceCandidate { + let _webrtc = values[0] + .get::() + .expect("Invalid argument"); + let sdp_mline_index = values[1].get::().expect("Invalid argument"); + let candidate = values[2].get::().expect("Invalid argument"); + + IceCandidate { + sdp_mline_index, + candidate, + } +} + +fn register_data_channel( + registry: Arc>>, + id: DataChannelId, + channel: GStreamerWebRtcDataChannel, +) -> WebRtcDataChannelResult { + if registry.lock().unwrap().contains_key(&id) { + return Err(WebRtcError::Backend( + "Could not register data channel. ID collision".to_owned(), + )); + } + registry + .lock() + .unwrap() + .insert(id, DataChannelEventTarget::Created(channel)); + Ok(id) +} diff --git a/components/media/backends/ohos/Cargo.toml b/components/media/backends/ohos/Cargo.toml new file mode 100644 index 00000000000..e130a1fd069 --- /dev/null +++ b/components/media/backends/ohos/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "servo-media-ohos" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_ohos" +path = "lib.rs" + +[dependencies] +servo-media = { path = "../../servo-media" } +servo-media-audio = { path = "../../audio" } +servo-media-player = { path = "../../player" } +servo-media-streams = { path = "../../streams" } +servo-media-traits = { path = "../../traits" } +servo-media-webrtc = { path = "../../webrtc" } +mime = "0.3.13" +once_cell = "1.18.0" +log = "0.4" +ohos-media-sys = { version = "0.0.5" ,features = ["api-21"] } diff --git a/components/media/backends/ohos/lib.rs b/components/media/backends/ohos/lib.rs new file mode 100644 index 00000000000..3b14efc5646 --- /dev/null +++ b/components/media/backends/ohos/lib.rs @@ -0,0 +1,208 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::{ + collections::HashMap, + sync::{ + atomic::AtomicUsize, + mpsc::{self, Sender}, + Arc, Mutex, Weak, + }, + thread, +}; + +use log::warn; +use mime::Mime; +use servo_media::{ + Backend, BackendInit, BackendMsg, ClientContextId, MediaInstance, SupportsMediaType, +}; + +use crate::{player::OhosAVPlayer, registry_scanner::OHOS_REGISTRY_SCANNER}; +mod player; +mod registry_scanner; + +pub struct OhosBackend { + instances: Arc>)>>>>, + next_instance_id: AtomicUsize, + backend_chan: Arc>>, +} + +impl OhosBackend { + fn media_instance_action( + &self, + id: &ClientContextId, + cb: &dyn Fn(&dyn MediaInstance) -> Result<(), ()>, + ) { + let mut instances = self.instances.lock().unwrap(); + match instances.get_mut(id) { + Some(vec) => vec.retain(|(_, weak)| { + if let Some(instance) = weak.upgrade() { + if cb(&*(instance.lock().unwrap())).is_err() { + warn!("Error executing media instance action"); + } + true + } else { + false + } + }), + None => { + warn!("Trying to exec media action on an unknown client context"); + } + } + } +} + +impl BackendInit for OhosBackend { + fn init() -> Box { + let instances: Arc< + Mutex>)>>>, + > = Arc::new(Mutex::new(HashMap::new())); + + let instances_ = instances.clone(); + let (backend_chan, recvr) = mpsc::channel(); + thread::Builder::new() + .name("OhosBackend ShutdownThread".to_owned()) + .spawn(move || { + match recvr.recv().unwrap() { + BackendMsg::Shutdown { context, id, tx_ack } => { + let mut map = instances_.lock().unwrap(); + if let Some(vec) = map.get_mut(&context) { + vec.retain(|m| m.0 != id); + if vec.is_empty() { + map.remove(&context); + } + } + let _ = tx_ack.send(()); + } + }; + }) + .unwrap(); + return Box::new(OhosBackend { + next_instance_id: AtomicUsize::new(0), + instances, + backend_chan: Arc::new(Mutex::new(backend_chan)), + }); + } +} + +// https://developer.huawei.com/consumer/en/doc/harmonyos-guides/obtain-supported-codecs +// https://developer.huawei.com/consumer/en/doc/harmonyos-guides/media-kit-intro-V5#supported-formats-and-protocols + +impl Backend for OhosBackend { + fn create_player( + &self, + id: &servo_media::ClientContextId, + stream_type: servo_media_player::StreamType, + sender: servo_media_player::ipc_channel::ipc::IpcSender, + video_renderer: Option< + std::sync::Arc>, + >, + audio_renderer: Option< + std::sync::Arc>, + >, + gl_context: Box, + ) -> std::sync::Arc> { + Arc::new(Mutex::new(OhosAVPlayer::new())) + } + + fn create_audiostream(&self) -> servo_media_streams::MediaStreamId { + todo!() + } + + fn create_videostream(&self) -> servo_media_streams::MediaStreamId { + todo!() + } + + fn create_stream_output(&self) -> Box { + todo!() + } + + fn create_stream_and_socket( + &self, + ty: servo_media_streams::MediaStreamType, + ) -> ( + Box, + servo_media_streams::MediaStreamId, + ) { + todo!() + } + + fn create_audioinput_stream( + &self, + set: servo_media_streams::capture::MediaTrackConstraintSet, + ) -> Option { + todo!() + } + + fn create_videoinput_stream( + &self, + set: servo_media_streams::capture::MediaTrackConstraintSet, + ) -> Option { + todo!() + } + + fn create_audio_context( + &self, + id: &servo_media::ClientContextId, + options: servo_media_audio::context::AudioContextOptions, + ) -> Result< + std::sync::Arc>, + servo_media_audio::sink::AudioSinkError, + > { + todo!() + } + + fn create_webrtc( + &self, + signaller: Box, + ) -> servo_media_webrtc::WebRtcController { + todo!() + } + + fn can_play_type(&self, media_type: &str) -> servo_media::SupportsMediaType { + if let Ok(mime) = media_type.parse::() { + let mime_type = mime.type_().as_str().to_owned() + "/" + mime.subtype().as_str(); + let codecs = match mime.get_param("codecs") { + Some(codecs) => codecs + .as_str() + .split(',') + .map(|codec| codec.trim()) + .collect(), + None => vec![], + }; + + if OHOS_REGISTRY_SCANNER.are_mime_and_codecs_supported(&mime_type, &codecs) { + if codecs.is_empty() { + return SupportsMediaType::Maybe; + } + return SupportsMediaType::Probably; + } + } + SupportsMediaType::No + } + + fn get_device_monitor( + &self, + ) -> Box { + todo!() + } + + fn mute(&self, id: &ClientContextId, val: bool) { + self.media_instance_action( + id, + &(move |instance: &dyn MediaInstance| instance.mute(val)), + ); + } + + fn resume(&self, id: &ClientContextId) { + self.media_instance_action(id, &(move |instance: &dyn MediaInstance| instance.resume())); + } + + fn suspend(&self, id: &ClientContextId) { + self.media_instance_action( + id, + &(move |instance: &dyn MediaInstance| instance.suspend()), + ); + } +} diff --git a/components/media/backends/ohos/player.rs b/components/media/backends/ohos/player.rs new file mode 100644 index 00000000000..b928c88aff7 --- /dev/null +++ b/components/media/backends/ohos/player.rs @@ -0,0 +1,130 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use servo_media::MediaInstance; +use servo_media_player::Player; + +pub struct OhosAVPlayer {} + +impl OhosAVPlayer { + pub fn new() -> OhosAVPlayer { + OhosAVPlayer {} + } +} + +impl MediaInstance for OhosAVPlayer { + fn get_id(&self) -> usize { + todo!() + } + + fn mute(&self, val: bool) -> Result<(), ()> { + todo!() + } + + fn suspend(&self) -> Result<(), ()> { + todo!() + } + + fn resume(&self) -> Result<(), ()> { + todo!() + } +} + +impl Player for OhosAVPlayer { + fn play(&self) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn pause(&self) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn paused(&self) -> bool { + todo!() + } + + fn can_resume(&self) -> bool { + todo!() + } + + fn stop(&self) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn seek(&self, time: f64) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn seekable(&self) -> Vec> { + todo!() + } + + fn set_mute(&self, muted: bool) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn muted(&self) -> bool { + todo!() + } + + fn set_volume(&self, volume: f64) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn volume(&self) -> f64 { + todo!() + } + + fn set_input_size(&self, size: u64) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn set_playback_rate(&self, playback_rate: f64) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn playback_rate(&self) -> f64 { + todo!() + } + + fn push_data(&self, data: Vec) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn end_of_stream(&self) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn buffered(&self) -> Vec> { + todo!() + } + + fn set_stream( + &self, + stream: &servo_media_streams::MediaStreamId, + only_stream: bool, + ) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn render_use_gl(&self) -> bool { + todo!() + } + + fn set_audio_track( + &self, + stream_index: i32, + enabled: bool, + ) -> Result<(), servo_media_player::PlayerError> { + todo!() + } + + fn set_video_track( + &self, + stream_index: i32, + enabled: bool, + ) -> Result<(), servo_media_player::PlayerError> { + todo!() + } +} diff --git a/components/media/backends/ohos/registry_scanner.rs b/components/media/backends/ohos/registry_scanner.rs new file mode 100644 index 00000000000..7b1d645738d --- /dev/null +++ b/components/media/backends/ohos/registry_scanner.rs @@ -0,0 +1,75 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::collections::HashMap; + +use once_cell::sync::Lazy; + +pub static OHOS_REGISTRY_SCANNER: Lazy = + Lazy::new(|| OhosRegistryScanner::new()); + +// Should be a combination of mime/codecs +// If the type we are matching only contain mime, then we only match the container. +// +pub struct OhosRegistryScanner { + av_player_supported_mime_codecs_type: HashMap<&'static str, &'static [&'static str]>, +} + +impl OhosRegistryScanner { + fn new() -> OhosRegistryScanner { + let mut registry_scanner = OhosRegistryScanner { + av_player_supported_mime_codecs_type: HashMap::new(), + }; + registry_scanner.initialize_av_player_container_and_codecs(); + registry_scanner + } + + pub fn are_mime_and_codecs_supported(&self, container_type: &str, codecs: &Vec<&str>) -> bool { + let Some(supported_codecs) = self + .av_player_supported_mime_codecs_type + .get(container_type) + else { + return false; + }; + codecs.iter().all(|codec| { + supported_codecs.contains(codec) || { + supported_codecs.iter().any(|supported_codec| { + if let Some(stripped) = supported_codec.strip_suffix('*') { + if codec.starts_with(stripped) { + return true; + } + } + false + }) + } + }) + } + + fn initialize_av_player_container_and_codecs(&mut self) { + // Video Container + self.av_player_supported_mime_codecs_type + .insert("video/mp4", &["hev1*", "hvc1*", "aac", "mp3", "avc*"]); + self.av_player_supported_mime_codecs_type + .insert("video/mkv", &["hev1*", "hvc1*", "aac", "mp3", "avc*"]); + self.av_player_supported_mime_codecs_type + .insert("video/ts", &["hev1*", "hvc1*", "aac", "mp3", "avc*"]); + // Audio Container + self.av_player_supported_mime_codecs_type + .insert("audio/m4a", &["aac"]); + self.av_player_supported_mime_codecs_type + .insert("audio/aac", &["aac"]); + self.av_player_supported_mime_codecs_type + .insert("audio/mp3", &["mp3"]); + self.av_player_supported_mime_codecs_type + .insert("audio/ogg", &["vorbis"]); + self.av_player_supported_mime_codecs_type + .insert("audio/wav", &["1", "audio/pcm"]); + self.av_player_supported_mime_codecs_type + .insert("audio/flac", &["flac"]); + self.av_player_supported_mime_codecs_type + .insert("audio/amr", &["amr"]); + self.av_player_supported_mime_codecs_type + .insert("audio/ape", &["ape"]); + } +} diff --git a/components/media/examples/Cargo.toml b/components/media/examples/Cargo.toml new file mode 100644 index 00000000000..a5059e1dc1f --- /dev/null +++ b/components/media/examples/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "media-examples" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +euclid = { workspace = true } +rand = { workspace = true } +serde = { workspace = true } +servo-media = { workspace = true } +servo-media-auto = { path = "../backends/auto" } +servo-media-dummy = { path = "../backends/dummy" } diff --git a/components/media/examples/examples/audioinput_stream.rs b/components/media/examples/examples/audioinput_stream.rs new file mode 100644 index 00000000000..f76f8360f4f --- /dev/null +++ b/components/media/examples/examples/audioinput_stream.rs @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::ServoMedia; + +fn run_example(servo_media: Arc) { + if let Some(stream) = servo_media.create_audioinput_stream(Default::default()) { + let mut output = servo_media.create_stream_output(); + output.add_stream(&stream); + thread::sleep(time::Duration::from_millis(6000)); + } else { + print!("No audio input elements available"); + } +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/biquad.rs b/components/media/examples/examples/biquad.rs new file mode 100644 index 00000000000..255488a4a04 --- /dev/null +++ b/components/media/examples/examples/biquad.rs @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::biquad_filter_node::{ + BiquadFilterNodeMessage, BiquadFilterNodeOptions, FilterType, +}; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let mut options = OscillatorNodeOptions::default(); + options.freq = 100.; + let osc1 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + options.freq = 800.; + let osc2 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let mut options = BiquadFilterNodeOptions::default(); + options.frequency = 50.; + options.filter = FilterType::LowPass; + let biquad = context + .create_node(AudioNodeInit::BiquadFilterNode(options), Default::default()) + .expect("Failed to create biquad filter node"); + context.connect_ports(osc1.output(0), biquad.input(0)); + context.connect_ports(osc2.output(0), biquad.input(0)); + context.connect_ports(biquad.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc1, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc2, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + biquad, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 1000., 2.), + ), + ); + + thread::sleep(time::Duration::from_millis(2200)); + context.message_node( + biquad, + AudioNodeMessage::BiquadFilterNode(BiquadFilterNodeMessage::SetFilterType( + FilterType::BandPass, + )), + ); + + thread::sleep(time::Duration::from_millis(1000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/channels.rs b/components/media/examples/examples/channels.rs new file mode 100644 index 00000000000..bed56e059bd --- /dev/null +++ b/components/media/examples/examples/channels.rs @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::channel_node::ChannelNodeOptions; +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let mut options = OscillatorNodeOptions::default(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + options.freq = 213.; + let osc2 = context + .create_node(AudioNodeInit::OscillatorNode(options), Default::default()) + .expect("Failed to create oscillator node"); + let mut options = GainNodeOptions::default(); + options.gain = 0.7; + let gain = context + .create_node(AudioNodeInit::GainNode(options.clone()), Default::default()) + .expect("Failed to create gain node"); + let options = ChannelNodeOptions { channels: 2 }; + let merger = context + .create_node( + AudioNodeInit::ChannelMergerNode(options), + Default::default(), + ) + .expect("Failed to create channel merger node"); + + let dest = context.dest_node(); + context.connect_ports(osc.output(0), gain.input(0)); + context.connect_ports(gain.output(0), merger.input(0)); + context.connect_ports(osc2.output(0), merger.input(1)); + context.connect_ports(merger.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc2, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + let _ = context.resume(); + + thread::sleep(time::Duration::from_millis(2000)); + context.message_node(dest, AudioNodeMessage::SetChannelCount(1)); + thread::sleep(time::Duration::from_millis(2000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/channelsum.rs b/components/media/examples/examples/channelsum.rs new file mode 100644 index 00000000000..3a3584f9af3 --- /dev/null +++ b/components/media/examples/examples/channelsum.rs @@ -0,0 +1,84 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::channel_node::ChannelNodeOptions; +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let mut options = OscillatorNodeOptions::default(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + options.freq = 213.; + let osc2 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + options.freq = 100.; + let osc3 = context + .create_node(AudioNodeInit::OscillatorNode(options), Default::default()) + .expect("Failed to create oscillator node"); + let mut options = GainNodeOptions::default(); + options.gain = 0.7; + let gain = context + .create_node(AudioNodeInit::GainNode(options.clone()), Default::default()) + .expect("Failed to create gain node"); + + let options = ChannelNodeOptions { channels: 2 }; + let merger = context + .create_node( + AudioNodeInit::ChannelMergerNode(options), + Default::default(), + ) + .expect("Failed to create channel merger node"); + + let dest = context.dest_node(); + context.connect_ports(osc.output(0), merger.input(0)); + context.connect_ports(osc2.output(0), merger.input(1)); + context.connect_ports(merger.output(0), gain.input(0)); + context.connect_ports(osc3.output(0), gain.input(0)); + context.connect_ports(gain.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc2, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc3, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + let _ = context.resume(); + + thread::sleep(time::Duration::from_millis(2000)); + context.message_node(dest, AudioNodeMessage::SetChannelCount(1)); + thread::sleep(time::Duration::from_millis(2000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/constant_source.rs b/components/media/examples/examples/constant_source.rs new file mode 100644 index 00000000000..b30bbfd4c99 --- /dev/null +++ b/components/media/examples/examples/constant_source.rs @@ -0,0 +1,107 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::constant_source_node::ConstantSourceNodeOptions; +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + + let cs = context + .create_node( + AudioNodeInit::ConstantSourceNode(ConstantSourceNodeOptions::default()), + Default::default(), + ) + .expect("Failed to create ConstantSourceNode node"); + + let mut gain_options = GainNodeOptions::default(); + gain_options.gain = 0.1; + let gain = context + .create_node( + AudioNodeInit::GainNode(gain_options.clone()), + Default::default(), + ) + .expect("Failed to create gain node"); + + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + + context.connect_ports(osc.output(0), gain.input(0)); + context.connect_ports(cs.output(0), gain.param(ParamType::Gain)); + context.connect_ports(gain.output(0), dest.input(0)); + + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + context.message_node( + gain, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + context.message_node( + cs, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + context.message_node( + cs, + AudioNodeMessage::SetParam( + ParamType::Offset, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 1., 1.5), + ), + ); + + context.message_node( + cs, + AudioNodeMessage::SetParam( + ParamType::Offset, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 0.1, 3.0), + ), + ); + + context.message_node( + cs, + AudioNodeMessage::SetParam( + ParamType::Offset, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 1., 4.5), + ), + ); + + context.message_node( + cs, + AudioNodeMessage::SetParam( + ParamType::Offset, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 0.1, 6.0), + ), + ); + + thread::sleep(time::Duration::from_millis(9000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/dummy.rs b/components/media/examples/examples/dummy.rs new file mode 100644 index 00000000000..e8ad437514c --- /dev/null +++ b/components/media/examples/examples/dummy.rs @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_dummy; + +use servo_media::ServoMedia; +use servo_media_dummy::DummyBackend; + +fn main() { + ServoMedia::init::(); + ServoMedia::get(); +} diff --git a/components/media/examples/examples/iir_filter.rs b/components/media/examples/examples/iir_filter.rs new file mode 100644 index 00000000000..05c09e483a4 --- /dev/null +++ b/components/media/examples/examples/iir_filter.rs @@ -0,0 +1,89 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::iir_filter_node::{IIRFilterNode, IIRFilterNodeOptions}; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let id = ClientContextId::build(1, 1); + let context = servo_media + .create_audio_context(&id, Default::default()) + .unwrap(); + + { + let context = context.lock().unwrap(); + + let dest = context.dest_node(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(OscillatorNodeOptions::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + + let feedback = Arc::new(vec![7.0, 1.0, 1.0]); + let feedforward = Arc::new(vec![1.0, 1.0, 1.0]); + + let iir = context + .create_node( + AudioNodeInit::IIRFilterNode(IIRFilterNodeOptions { + feedback: feedback.clone(), + feedforward: feedforward.clone(), + }), + Default::default(), + ) + .expect("Failed to create IIR filter node"); + + context.connect_ports(osc.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + println!("raw oscillator"); + thread::sleep(time::Duration::from_millis(2000)); + + println!("oscillator through iir filter"); + context.disconnect_output(osc.output(0)); + context.connect_ports(osc.output(0), iir.input(0)); + context.connect_ports(iir.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(2000)); + + println!("raw oscillator"); + context.disconnect_output(osc.output(0)); + context.disconnect_output(iir.output(0)); + context.connect_ports(osc.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(2000)); + + let freqs = vec![0.0, 0.3, 0.5]; + let mut mag = vec![0.0; 3]; + let mut phase = vec![0.0; 3]; + IIRFilterNode::get_frequency_response( + &feedforward, + &feedback, + &freqs, + &mut mag, + &mut phase, + ); + print!( + "GetFrequencyResponse for freqs: {:?}\n mag: {:?}\n phase: {:?}", + &freqs, &mag, &phase + ); + } +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/muted_audiocontext.rs b/components/media/examples/examples/muted_audiocontext.rs new file mode 100644 index 00000000000..0be9af10205 --- /dev/null +++ b/components/media/examples/examples/muted_audiocontext.rs @@ -0,0 +1,95 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::oscillator_node::OscillatorType::Sawtooth; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context_id1 = &ClientContextId::build(1, 1); + let context1 = servo_media.create_audio_context(&context_id1, Default::default()); + { + let context1 = context1.unwrap(); + let context = context1.lock().unwrap(); + let dest = context.dest_node(); + let options = OscillatorNodeOptions::default(); + let osc1 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc1.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc1, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + } + + let context_id2 = &ClientContextId::build(1, 3); + let context2 = servo_media.create_audio_context(&context_id2, Default::default()); + { + let mut options = OscillatorNodeOptions::default(); + options.oscillator_type = Sawtooth; + let context2 = context2.unwrap(); + let context = context2.lock().unwrap(); + let dest = context.dest_node(); + let osc3 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc3.output(0), dest.input(0)); + + let _ = context.resume(); + context.message_node( + osc3, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + } + + println!("servo_media raw s1"); + servo_media.mute(&context_id2, true); + thread::sleep(time::Duration::from_millis(2000)); + + println!("servo_media raw s2"); + servo_media.mute(&context_id1, true); + servo_media.mute(&context_id2, false); + thread::sleep(time::Duration::from_millis(2000)); + + println!("servo_media s1+s2"); + servo_media.mute(&context_id1, false); + thread::sleep(time::Duration::from_millis(2000)); + + println!("servo_media muting s1"); + servo_media.mute(&context_id1, true); + thread::sleep(time::Duration::from_millis(2000)); + + println!("servo_media muting s2"); + servo_media.mute(&context_id2, true); + thread::sleep(time::Duration::from_millis(2000)); + + println!("servo_media unmuting s2"); + servo_media.mute(&context_id2, false); + thread::sleep(time::Duration::from_millis(2000)); + + println!("servo_media unmuting s1"); + servo_media.mute(&context_id1, false); + thread::sleep(time::Duration::from_millis(2000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/offline_context.rs b/components/media/examples/examples/offline_context.rs new file mode 100644 index 00000000000..06348843dcb --- /dev/null +++ b/components/media/examples/examples/offline_context.rs @@ -0,0 +1,89 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::{Arc, Mutex, mpsc}; +use std::{thread, time}; + +use servo_media::audio::block::FRAMES_PER_BLOCK_USIZE; +use servo_media::audio::buffer_source_node::{AudioBuffer, AudioBufferSourceNodeMessage}; +use servo_media::audio::context::{AudioContextOptions, OfflineAudioContextOptions}; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + // Create offline context to process 1024 blocks of a oscillator node produced + // sine wave. + let mut options = ::default(); + options.channels = 2; + options.length = 1024 * FRAMES_PER_BLOCK_USIZE; + let sample_rate = options.sample_rate; + let options = AudioContextOptions::OfflineAudioContext(options); + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), options) + .unwrap(); + let context = context.lock().unwrap(); + let processed_audio = Arc::new(Mutex::new(Vec::new())); + let processed_audio_ = processed_audio.clone(); + let (sender, receiver) = mpsc::channel(); + let sender = Mutex::new(sender); + context.set_eos_callback(Box::new(move |buffer| { + processed_audio + .lock() + .unwrap() + .extend_from_slice((*buffer).as_ref()); + sender.lock().unwrap().send(()).unwrap(); + })); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let dest = context.dest_node(); + context.connect_ports(osc.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + let _ = context.resume(); + // Block until we processed the data. + receiver.recv().unwrap(); + // Close offline context. + let _ = context.close(); + // Create audio context to play the processed audio. + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 2), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let buffer_source = context + .create_node( + AudioNodeInit::AudioBufferSourceNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let dest = context.dest_node(); + context.connect_ports(buffer_source.output(0), dest.input(0)); + context.message_node( + buffer_source, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + buffer_source, + AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer(Some( + AudioBuffer::from_buffer(processed_audio_.lock().unwrap().to_vec(), sample_rate), + ))), + ); + let _ = context.resume(); + thread::sleep(time::Duration::from_millis(5000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/oscillator.rs b/components/media/examples/examples/oscillator.rs new file mode 100644 index 00000000000..5a26d72545f --- /dev/null +++ b/components/media/examples/examples/oscillator.rs @@ -0,0 +1,143 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::oscillator_node::OscillatorType::{Custom, Sawtooth, Square, Triangle}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let mut options = OscillatorNodeOptions::default(); + let osc1 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc1.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc1, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); + thread::sleep(time::Duration::from_millis(3000)); + + options.oscillator_type = Square; + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 2), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc2 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc2.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc2, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); + thread::sleep(time::Duration::from_millis(1000)); + + options.oscillator_type = Sawtooth; + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 3), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc3 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc3.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(3000)); + + let _ = context.resume(); + context.message_node( + osc3, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); + thread::sleep(time::Duration::from_millis(1000)); + + options.oscillator_type = Triangle; + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 4), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc4 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc4.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(3000)); + + let _ = context.resume(); + context.message_node( + osc4, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); + + thread::sleep(time::Duration::from_millis(3000)); + + options.oscillator_type = Custom; + + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 5), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc5 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc5.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(3000)); + + let _ = context.resume(); + context.message_node( + osc4, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); + thread::sleep(time::Duration::from_millis(1000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/panner.rs b/components/media/examples/examples/panner.rs new file mode 100644 index 00000000000..6fc279c2393 --- /dev/null +++ b/components/media/examples/examples/panner.rs @@ -0,0 +1,190 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::panner_node::PannerNodeOptions; +use servo_media::audio::param::{ParamDir, ParamType, RampKind, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let listener = context.listener(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let mut options = PannerNodeOptions::default(); + options.cone_outer_angle = 0.; + options.position_x = 100.; + options.position_y = 0.; + options.position_z = 100.; + options.ref_distance = 100.; + options.rolloff_factor = 0.01; + let panner = context + .create_node(AudioNodeInit::PannerNode(options), Default::default()) + .expect("Failed to create panner node"); + context.connect_ports(osc.output(0), panner.input(0)); + context.connect_ports(panner.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + // trace a square around your head twice + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 0.2), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 0.2), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 0.4), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 0.4), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 0.6), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 0.6), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 0.8), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 0.8), + ), + ); + + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 1.0), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 1.0), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 1.2), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 1.2), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 1.4), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, -100., 1.4), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::X), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 1.6), + ), + ); + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 100., 1.6), + ), + ); + // now it runs away + context.message_node( + panner, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 10000., 3.), + ), + ); + context.message_node( + listener, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::SetValueAtTime(0., 3.), + ), + ); + // chase it + context.message_node( + listener, + AudioNodeMessage::SetParam( + ParamType::Position(ParamDir::Z), + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 10000., 4.), + ), + ); + thread::sleep(time::Duration::from_millis(4000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/params.rs b/components/media/examples/examples/params.rs new file mode 100644 index 00000000000..b8b1c0747d4 --- /dev/null +++ b/components/media/examples/examples/params.rs @@ -0,0 +1,112 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let mut options = GainNodeOptions::default(); + options.gain = 0.5; + let gain = context + .create_node(AudioNodeInit::GainNode(options), Default::default()) + .expect("Failed to create gain node"); + context.connect_ports(osc.output(0), gain.input(0)); + context.connect_ports(gain.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + // 0.5s: Set frequency to 110Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetValueAtTime(110., 0.5), + ), + ); + // 1s: Set frequency to 220Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetValueAtTime(220., 1.), + ), + ); + // 0.75s: Set gain to 0.25 + context.message_node( + gain, + AudioNodeMessage::SetParam( + ParamType::Gain, + UserAutomationEvent::SetValueAtTime(0.25, 0.75), + ), + ); + // 0.75s - 1.5s: Exponentially ramp gain to 1 + context.message_node( + gain, + AudioNodeMessage::SetParam( + ParamType::Gain, + UserAutomationEvent::RampToValueAtTime(RampKind::Exponential, 1., 1.5), + ), + ); + // 0.75s - 1.75s: Linearly ramp frequency to 880Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 880., 1.75), + ), + ); + // 1.75s - 2.5s: Exponentially ramp frequency to 110Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::RampToValueAtTime(RampKind::Exponential, 110., 2.5), + ), + ); + + // 2.75s: Exponentially approach 110Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetTargetAtTime(1100., 2.75, 1.1), + ), + ); + // 3.3s: But actually stop at 3.3Hz and hold + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::CancelAndHoldAtTime(3.3), + ), + ); + thread::sleep(time::Duration::from_millis(5000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/params_connect.rs b/components/media/examples/examples/params_connect.rs new file mode 100644 index 00000000000..7d5154443ab --- /dev/null +++ b/components/media/examples/examples/params_connect.rs @@ -0,0 +1,69 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let mut options = OscillatorNodeOptions::default(); + options.freq = 2.0; + let lfo = context + .create_node(AudioNodeInit::OscillatorNode(options), Default::default()) + .expect("Failed to create oscillator node"); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let gain = context + .create_node( + AudioNodeInit::GainNode(Default::default()), + Default::default(), + ) + .expect("Failed to create gain node"); + let dest = context.dest_node(); + context.connect_ports(lfo.output(0), gain.param(ParamType::Gain)); + context.connect_ports(gain.output(0), dest.input(0)); + context.connect_ports(osc.output(0), gain.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + lfo, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + thread::sleep(time::Duration::from_millis(3000)); + // 0.75s - 1.75s: Linearly ramp frequency to 880Hz + context.message_node( + gain, + AudioNodeMessage::SetParam( + ParamType::Gain, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 0., 6.), + ), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/params_connect2.rs b/components/media/examples/examples/params_connect2.rs new file mode 100644 index 00000000000..9bc286fe99c --- /dev/null +++ b/components/media/examples/examples/params_connect2.rs @@ -0,0 +1,69 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let mut options = OscillatorNodeOptions::default(); + options.freq = 2.0; + let lfo = context + .create_node(AudioNodeInit::OscillatorNode(options), Default::default()) + .expect("Failed to create oscillator node"); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let mut options = GainNodeOptions::default(); + options.gain = 100.; + let gain = context + .create_node(AudioNodeInit::GainNode(options), Default::default()) + .expect("Failed to create gain node"); + let dest = context.dest_node(); + context.connect_ports(lfo.output(0), gain.input(0)); + context.connect_ports(gain.output(0), osc.param(ParamType::Frequency)); + context.connect_ports(osc.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + lfo, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + thread::sleep(time::Duration::from_millis(3000)); + // 0.75s - 1.75s: Linearly ramp frequency to 880Hz + context.message_node( + gain, + AudioNodeMessage::SetParam( + ParamType::Gain, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 0., 6.), + ), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/params_settarget.rs b/components/media/examples/examples/params_settarget.rs new file mode 100644 index 00000000000..77ee22b9842 --- /dev/null +++ b/components/media/examples/examples/params_settarget.rs @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + context.connect_ports(osc.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + // 0.1s: Set frequency to 110Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetValueAtTime(110., 0.1), + ), + ); + // 0.3s: Start increasing frequency to 440Hz exponentially with a time constant of 1 + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetTargetAtTime(440., 0.3, 1.), + ), + ); + // 1.5s: Start increasing frequency to 1760Hz exponentially + // this event effectively doesn't happen, but instead sets a starting point + // for the next ramp event + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetTargetAtTime(1760., 1.5, 0.1), + ), + ); + // 1.5s - 3s Linearly ramp down from the previous event (1.5s) to 110Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 110., 3.0), + ), + ); + thread::sleep(time::Duration::from_millis(5000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/play.rs b/components/media/examples/examples/play.rs new file mode 100644 index 00000000000..790347ffa73 --- /dev/null +++ b/components/media/examples/examples/play.rs @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{ + AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage, OnEndedCallback, +}; +use servo_media::audio::param::{ParamType, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let mut options = GainNodeOptions::default(); + options.gain = 0.5; + let gain = context + .create_node(AudioNodeInit::GainNode(options), Default::default()) + .expect("Failed to create gain node"); + let dest = context.dest_node(); + context.connect_ports(osc.output(0), gain.input(0)); + context.connect_ports(gain.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Stop(3.)), + ); + let callback = OnEndedCallback::new(|| { + println!("Playback ended"); + }); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode( + AudioScheduledSourceNodeMessage::RegisterOnEndedCallback(callback), + ), + ); + assert_eq!(context.current_time(), 0.); + let _ = context.resume(); + // 0.5s: Set frequency to 110Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetValueAtTime(110., 0.5), + ), + ); + // 1s: Set frequency to 220Hz + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetValueAtTime(220., 1.), + ), + ); + // 0.75s: Set gain to 0.25 + context.message_node( + gain, + AudioNodeMessage::SetParam( + ParamType::Gain, + UserAutomationEvent::SetValueAtTime(0.25, 0.75), + ), + ); + thread::sleep(time::Duration::from_millis(1200)); + // 1.2s: Suspend processing + let _ = context.suspend(); + thread::sleep(time::Duration::from_millis(500)); + // 1.7s: Resume processing + let _ = context.resume(); + let current_time = context.current_time(); + assert!(current_time > 0.); + // Leave some time to enjoy the silence after stopping the + // oscillator node. + thread::sleep(time::Duration::from_millis(5000)); + // And check that we keep incrementing playback time. + assert!(current_time < context.current_time()); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/play_noise.rs b/components/media/examples/examples/play_noise.rs new file mode 100644 index 00000000000..4c5fa506c1d --- /dev/null +++ b/components/media/examples/examples/play_noise.rs @@ -0,0 +1,64 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate rand; +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::buffer_source_node::{AudioBuffer, AudioBufferSourceNodeMessage}; +use servo_media::audio::node::{ + AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage, OnEndedCallback, +}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let buffer_source = context + .create_node( + AudioNodeInit::AudioBufferSourceNode(Default::default()), + Default::default(), + ) + .expect("Failed to create buffer source node"); + let dest = context.dest_node(); + context.connect_ports(buffer_source.output(0), dest.input(0)); + let mut buffers = vec![Vec::with_capacity(4096), Vec::with_capacity(4096)]; + for _ in 0..4096 { + buffers[0].push(rand::random::()); + buffers[1].push(rand::random::()); + } + context.message_node( + buffer_source, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + buffer_source, + AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer(Some( + AudioBuffer::from_buffers(buffers, 44100.), + ))), + ); + let callback = OnEndedCallback::new(|| { + println!("Playback ended"); + }); + context.message_node( + buffer_source, + AudioNodeMessage::AudioScheduledSourceNode( + AudioScheduledSourceNodeMessage::RegisterOnEndedCallback(callback), + ), + ); + let _ = context.resume(); + thread::sleep(time::Duration::from_millis(5000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/resources/mov_bbb.mp4 b/components/media/examples/examples/resources/mov_bbb.mp4 new file mode 100644 index 00000000000..98f8be66040 Binary files /dev/null and b/components/media/examples/examples/resources/mov_bbb.mp4 differ diff --git a/components/media/examples/examples/resources/viper_cut.ogg b/components/media/examples/examples/resources/viper_cut.ogg new file mode 100644 index 00000000000..3ea45e710cb Binary files /dev/null and b/components/media/examples/examples/resources/viper_cut.ogg differ diff --git a/components/media/examples/examples/set_value_curve.rs b/components/media/examples/examples/set_value_curve.rs new file mode 100644 index 00000000000..ce9385259d3 --- /dev/null +++ b/components/media/examples/examples/set_value_curve.rs @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::constant_source_node::ConstantSourceNodeOptions; +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::param::{ParamType, UserAutomationEvent}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + + // Initializing the values vector for SetValueCurve function + let values = vec![ + 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., + ]; + let start_time = 0.; + let end_time = 5.; + let n = values.len() as f32; + let value_next = values[(n - 1.) as usize]; + + let cs = context + .create_node( + AudioNodeInit::ConstantSourceNode(ConstantSourceNodeOptions::default()), + Default::default(), + ) + .expect("Failed to create ConstantSourceNode node"); + + let mut gain_options = GainNodeOptions::default(); + gain_options.gain = 0.0; + let gain = context + .create_node( + AudioNodeInit::GainNode(gain_options.clone()), + Default::default(), + ) + .expect("Failed to create gain node"); + + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + + context.connect_ports(osc.output(0), gain.input(0)); + context.connect_ports(cs.output(0), gain.param(ParamType::Gain)); + context.connect_ports(gain.output(0), dest.input(0)); + + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + context.message_node( + gain, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + context.message_node( + cs, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + context.message_node( + cs, + AudioNodeMessage::SetParam( + ParamType::Offset, + UserAutomationEvent::SetValueCurveAtTime(values, start_time, end_time), + ), + ); + + context.message_node( + osc, + AudioNodeMessage::SetParam( + ParamType::Frequency, + UserAutomationEvent::SetValueAtTime(value_next, end_time), + ), + ); + + thread::sleep(time::Duration::from_millis(7000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/stereo_panner.rs b/components/media/examples/examples/stereo_panner.rs new file mode 100644 index 00000000000..a2a19373499 --- /dev/null +++ b/components/media/examples/examples/stereo_panner.rs @@ -0,0 +1,68 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::audio::stereo_panner::StereoPannerOptions; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(Default::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let mut options = StereoPannerOptions::default(); + options.pan = 0.; + let pan = context + .create_node(AudioNodeInit::StereoPannerNode(options), Default::default()) + .expect("Failed to create stereo panner node"); + context.connect_ports(osc.output(0), pan.input(0)); + context.connect_ports(pan.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + // 2s: Set pan to -1 + context.message_node( + pan, + AudioNodeMessage::SetParam(ParamType::Pan, UserAutomationEvent::SetValueAtTime(-1., 2.)), + ); + // 4s: Linearly ramp pan to 0 + context.message_node( + pan, + AudioNodeMessage::SetParam( + ParamType::Pan, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 0., 4.), + ), + ); + // 6s: Linearly ramp pan to 1 + context.message_node( + pan, + AudioNodeMessage::SetParam( + ParamType::Pan, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 1., 6.), + ), + ); + thread::sleep(time::Duration::from_millis(5000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/stream_dest_node.rs b/components/media/examples/examples/stream_dest_node.rs new file mode 100644 index 00000000000..40ec057eeff --- /dev/null +++ b/components/media/examples/examples/stream_dest_node.rs @@ -0,0 +1,55 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::streams::MediaStreamType; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = servo_media + .create_audio_context(&ClientContextId::build(1, 1), Default::default()) + .unwrap(); + let context = context.lock().unwrap(); + let options = OscillatorNodeOptions::default(); + let osc1 = context + .create_node( + AudioNodeInit::OscillatorNode(options.clone()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + + let (socket, id) = servo_media.create_stream_and_socket(MediaStreamType::Audio); + let dest = context + .create_node( + AudioNodeInit::MediaStreamDestinationNode(socket), + Default::default(), + ) + .expect("Failed to create stream destination node"); + context.connect_ports(osc1.output(0), dest.input(0)); + + let mut output = servo_media.create_stream_output(); + output.add_stream(&id); + let _ = context.resume(); + context.message_node( + osc1, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); + thread::sleep(time::Duration::from_millis(1000)); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/stream_reader_node.rs b/components/media/examples/examples/stream_reader_node.rs new file mode 100644 index 00000000000..3aac8537415 --- /dev/null +++ b/components/media/examples/examples/stream_reader_node.rs @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::AudioNodeInit; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let context = + servo_media.create_audio_context(&ClientContextId::build(1, 1), Default::default()); + let input = servo_media.create_audiostream(); + let context = context.unwrap(); + let context = context.lock().unwrap(); + let dest = context.dest_node(); + let osc1 = context + .create_node( + AudioNodeInit::MediaStreamSourceNode(input), + Default::default(), + ) + .expect("Failed to create stream source node"); + context.connect_ports(osc1.output(0), dest.input(0)); + let _ = context.resume(); + + thread::sleep(time::Duration::from_millis(6000)); + let _ = context.close(); +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/videoinput_stream.rs b/components/media/examples/examples/videoinput_stream.rs new file mode 100644 index 00000000000..237e6ffc940 --- /dev/null +++ b/components/media/examples/examples/videoinput_stream.rs @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::ServoMedia; + +fn run_example(servo_media: Arc) { + if let Some(stream) = servo_media.create_videoinput_stream(Default::default()) { + let mut output = servo_media.create_stream_output(); + output.add_stream(&stream); + thread::sleep(time::Duration::from_millis(6000)); + } else { + print!("No video input elements available"); + } +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/examples/examples/wave_shaper.rs b/components/media/examples/examples/wave_shaper.rs new file mode 100644 index 00000000000..329f2024f52 --- /dev/null +++ b/components/media/examples/examples/wave_shaper.rs @@ -0,0 +1,112 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate servo_media; +extern crate servo_media_auto; + +use std::sync::Arc; +use std::{thread, time}; + +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::wave_shaper_node::{ + OverSampleType, WaveShaperNodeMessage, WaveShaperNodeOptions, +}; +use servo_media::{ClientContextId, ServoMedia}; + +fn run_example(servo_media: Arc) { + let id = ClientContextId::build(1, 1); + let context = servo_media.create_audio_context(&id, Default::default()); + + { + let context = context.unwrap(); + let context = context.lock().unwrap(); + let curve = vec![1., 0., 0., 0.75, 0.5]; + + let dest = context.dest_node(); + let osc = context + .create_node( + AudioNodeInit::OscillatorNode(OscillatorNodeOptions::default()), + Default::default(), + ) + .expect("Failed to create oscillator node"); + let wsh = context + .create_node( + AudioNodeInit::WaveShaperNode(WaveShaperNodeOptions { + curve: Some(curve.clone()), + oversample: OverSampleType::None, + }), + Default::default(), + ) + .expect("Failed to create waveshaper node"); + let wshx2 = context + .create_node( + AudioNodeInit::WaveShaperNode(WaveShaperNodeOptions { + curve: Some(curve.clone()), + oversample: OverSampleType::Double, + }), + Default::default(), + ) + .expect("Failed to create waveshaper node"); + let wshx4 = context + .create_node( + AudioNodeInit::WaveShaperNode(WaveShaperNodeOptions { + curve: Some(curve.clone()), + oversample: OverSampleType::Quadruple, + }), + Default::default(), + ) + .expect("Failed to create waveshaper node"); + + context.connect_ports(osc.output(0), dest.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + + println!("raw oscillator"); + thread::sleep(time::Duration::from_millis(2000)); + + println!("oscillator through waveshaper with no oversampling"); + context.disconnect_output(osc.output(0)); + context.connect_ports(osc.output(0), wsh.input(0)); + context.connect_ports(wsh.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(2000)); + + println!("oscillator through waveshaper with 2x oversampling"); + context.disconnect_output(osc.output(0)); + context.disconnect_output(wsh.output(0)); + context.connect_ports(osc.output(0), wshx2.input(0)); + context.connect_ports(wshx2.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(2000)); + + println!("oscillator through waveshaper with 4x oversampling"); + context.disconnect_output(osc.output(0)); + context.disconnect_output(wshx2.output(0)); + context.connect_ports(osc.output(0), wshx4.input(0)); + context.connect_ports(wshx4.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(2000)); + + println!("oscillator through waveshaper with no oversampling"); + context.disconnect_output(osc.output(0)); + context.disconnect_output(wshx4.output(0)); + context.connect_ports(osc.output(0), wsh.input(0)); + context.connect_ports(wsh.output(0), dest.input(0)); + thread::sleep(time::Duration::from_millis(2000)); + + println!("oscillator through waveshaper with no curve (should be same as raw oscillator)"); + context.message_node( + wsh, + AudioNodeMessage::WaveShaperNode(WaveShaperNodeMessage::SetCurve(None)), + ); + thread::sleep(time::Duration::from_millis(2000)); + } +} + +fn main() { + ServoMedia::init::(); + let servo_media = ServoMedia::get(); + run_example(servo_media); +} diff --git a/components/media/Cargo.toml b/components/media/media-thread/Cargo.toml similarity index 92% rename from components/media/Cargo.toml rename to components/media/media-thread/Cargo.toml index 90606389dee..25f10ee07d3 100644 --- a/components/media/Cargo.toml +++ b/components/media/media-thread/Cargo.toml @@ -19,5 +19,5 @@ paint_api = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } servo-media = { workspace = true } -servo_config = { path = "../config" } +servo_config = { path = "../../config" } webrender_api = { workspace = true } diff --git a/components/media/lib.rs b/components/media/media-thread/lib.rs similarity index 100% rename from components/media/lib.rs rename to components/media/media-thread/lib.rs diff --git a/components/media/media_thread.rs b/components/media/media-thread/media_thread.rs similarity index 100% rename from components/media/media_thread.rs rename to components/media/media-thread/media_thread.rs diff --git a/components/media/player/Cargo.toml b/components/media/player/Cargo.toml new file mode 100644 index 00000000000..b22cdc72692 --- /dev/null +++ b/components/media/player/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "servo-media-player" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_player" +path = "lib.rs" + +[dependencies] +serde = "1.0.66" +serde_derive = "1.0.66" +ipc-channel = { workspace = true } + +[dependencies.servo-media-streams] +path = "../streams" + +[dependencies.servo-media-traits] +path = "../traits" diff --git a/components/media/player/audio.rs b/components/media/player/audio.rs new file mode 100644 index 00000000000..af926988189 --- /dev/null +++ b/components/media/player/audio.rs @@ -0,0 +1,7 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +pub trait AudioRenderer: Send + 'static { + fn render(&mut self, sample: Box>, channel: u32); +} diff --git a/components/media/player/context.rs b/components/media/player/context.rs new file mode 100644 index 00000000000..2f4e1748ad8 --- /dev/null +++ b/components/media/player/context.rs @@ -0,0 +1,53 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! `PlayerGLContext` is a trait to be used to pass the GL context for +//! rendering purposes. +//! +//! The current consumer of this trait is the GL rendering mechanism +//! for the GStreamer backend. +//! +//! The client application should implement this trait and pass the +//! trait object to its `player` instance. + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum GlContext { + /// The EGL platform used primarily with the X11, Wayland and + /// Android window systems as well as on embedded Linux. + Egl(usize), + /// The GLX platform used primarily with the X11 window system. + Glx(usize), + Unknown, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum NativeDisplay { + /// The EGLDisplay memory address + Egl(usize), + /// XDisplay memory address + X11(usize), + /// wl_display memory address + Wayland(usize), + Headless, + Unknown, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum GlApi { + OpenGL, + OpenGL3, + Gles1, + Gles2, + None, +} + +pub trait PlayerGLContext { + /// Returns the GL context living pointer wrapped by `GlContext` + fn get_gl_context(&self) -> GlContext; + /// Returns the living pointer to the native display structure + /// wrapped by `NativeDisplay`. + fn get_native_display(&self) -> NativeDisplay; + /// Returns the GL API of the context + fn get_gl_api(&self) -> GlApi; +} diff --git a/components/media/player/lib.rs b/components/media/player/lib.rs new file mode 100644 index 00000000000..9e226245072 --- /dev/null +++ b/components/media/player/lib.rs @@ -0,0 +1,133 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::ops::Range; +use std::time::Duration; + +pub extern crate ipc_channel; +#[macro_use] +extern crate serde_derive; +extern crate servo_media_streams as streams; +extern crate servo_media_traits; + +pub mod audio; +pub mod context; +pub mod metadata; +pub mod video; + +use ipc_channel::ipc::{self, IpcSender}; +use servo_media_traits::MediaInstance; +use streams::registry::MediaStreamId; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum PlaybackState { + Stopped, + Buffering, + Paused, + Playing, +} + +#[derive(Debug, PartialEq)] +pub enum PlayerError { + /// Backend specific error. + Backend(String), + /// Could not push buffer contents to the player. + BufferPushFailed, + /// The player cannot consume more data. + EnoughData, + /// Setting End Of Stream failed. + EOSFailed, + /// The media stream is not seekable. + NonSeekableStream, + /// Tried to seek out of range. + SeekOutOfRange, + /// Setting an audio or video stream failed. + /// Possibly because the type of source is not PlayerSource::Stream. + SetStreamFailed, + // Setting an audio or video track failed. + SetTrackFailed, +} + +pub type SeekLockMsg = (bool, IpcSender<()>); + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SeekLock { + pub lock_channel: IpcSender, +} + +impl SeekLock { + pub fn unlock(&self, result: bool) { + let (ack_sender, ack_recv) = ipc::channel::<()>().expect("Could not create IPC channel"); + self.lock_channel.send((result, ack_sender)).unwrap(); + ack_recv.recv().unwrap() + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum PlayerEvent { + EndOfStream, + /// The player has enough data. The client should stop pushing data into. + EnoughData, + Error(String), + VideoFrameUpdated, + MetadataUpdated(metadata::Metadata), + // The `None` value means the duration is unknown, in which case this is likely a live stream. + DurationChanged(Option), + /// The internal player queue is running out of data. The client should start + /// pushing more data. + NeedData, + PositionChanged(f64), + /// The player needs the data to perform a seek to the given offset in bytes. + /// The next push_data should get the buffers from the new offset. + /// The player will be blocked until the user unlocks it through + /// the given SeekLock instance. + /// This event is only received for seekable stream types. + SeekData(u64, SeekLock), + /// The player has performed a seek to the given time offset in seconds. + SeekDone(f64), + StateChanged(PlaybackState), +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +pub enum StreamType { + /// No seeking is supported in the stream, such as a live stream. + Stream, + /// The stream is seekable. + Seekable, +} + +pub trait Player: Send + MediaInstance { + fn play(&self) -> Result<(), PlayerError>; + fn pause(&self) -> Result<(), PlayerError>; + fn paused(&self) -> bool; + fn can_resume(&self) -> bool; + fn stop(&self) -> Result<(), PlayerError>; + fn seek(&self, time: f64) -> Result<(), PlayerError>; + fn seekable(&self) -> Vec>; + fn set_mute(&self, muted: bool) -> Result<(), PlayerError>; + fn muted(&self) -> bool; + fn set_volume(&self, volume: f64) -> Result<(), PlayerError>; + fn volume(&self) -> f64; + fn set_input_size(&self, size: u64) -> Result<(), PlayerError>; + fn set_playback_rate(&self, playback_rate: f64) -> Result<(), PlayerError>; + fn playback_rate(&self) -> f64; + fn push_data(&self, data: Vec) -> Result<(), PlayerError>; + fn end_of_stream(&self) -> Result<(), PlayerError>; + /// Get the list of time ranges in seconds that have been buffered. + fn buffered(&self) -> Vec>; + /// Set the stream to be played by the player. + /// Only a single stream of the same type (audio or video) can be set. + /// Subsequent calls with a stream of the same type will override the previously + /// set stream. + /// This method requires the player to be constructed with StreamType::Stream. + /// It is important to give the correct value of `only_stream` indicating + /// that the audio or video stream being set is the only one expected. + /// Subsequent calls to `set_stream` after the `only_stream` flag has been + /// set to true will fail. + fn set_stream(&self, stream: &MediaStreamId, only_stream: bool) -> Result<(), PlayerError>; + /// If player's rendering draws using GL textures + fn render_use_gl(&self) -> bool; + fn set_audio_track(&self, stream_index: i32, enabled: bool) -> Result<(), PlayerError>; + fn set_video_track(&self, stream_index: i32, enabled: bool) -> Result<(), PlayerError>; +} diff --git a/components/media/player/metadata.rs b/components/media/player/metadata.rs new file mode 100644 index 00000000000..210bf61130b --- /dev/null +++ b/components/media/player/metadata.rs @@ -0,0 +1,20 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::time; + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +pub struct Metadata { + pub duration: Option, + pub width: u32, + pub height: u32, + pub format: String, + pub is_seekable: bool, + // TODO: Might be nice to move width and height along with each video track. + pub video_tracks: Vec, + pub audio_tracks: Vec, + // Whether the media comes from a live source or not. + pub is_live: bool, + pub title: Option, +} diff --git a/components/media/player/video.rs b/components/media/player/video.rs new file mode 100644 index 00000000000..2ebda45b3c0 --- /dev/null +++ b/components/media/player/video.rs @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::Arc; + +#[derive(Clone)] +pub enum VideoFrameData { + Raw(Arc>), + Texture(u32), + OESTexture(u32), +} + +pub trait Buffer: Send + Sync { + fn to_vec(&self) -> Option; +} + +#[derive(Clone)] +pub struct VideoFrame { + width: i32, + height: i32, + data: VideoFrameData, + _buffer: Arc, +} + +impl VideoFrame { + pub fn new(width: i32, height: i32, buffer: Arc) -> Option { + let data = buffer.to_vec()?; + Some(VideoFrame { + width, + height, + data, + _buffer: buffer, + }) + } + + pub fn get_width(&self) -> i32 { + self.width + } + + pub fn get_height(&self) -> i32 { + self.height + } + + pub fn get_data(&self) -> Arc> { + match self.data { + VideoFrameData::Raw(ref data) => data.clone(), + _ => unreachable!("invalid raw data request for texture frame"), + } + } + + pub fn get_texture_id(&self) -> u32 { + match self.data { + VideoFrameData::Texture(data) | VideoFrameData::OESTexture(data) => data, + _ => unreachable!("invalid texture id request for raw data frame"), + } + } + + pub fn is_gl_texture(&self) -> bool { + matches!( + self.data, + VideoFrameData::Texture(_) | VideoFrameData::OESTexture(_) + ) + } + + pub fn is_external_oes(&self) -> bool { + matches!(self.data, VideoFrameData::OESTexture(_)) + } +} + +pub trait VideoFrameRenderer: Send + 'static { + fn render(&mut self, frame: VideoFrame); +} diff --git a/components/media/servo-media-derive/Cargo.toml b/components/media/servo-media-derive/Cargo.toml new file mode 100644 index 00000000000..0f652ce6ccd --- /dev/null +++ b/components/media/servo-media-derive/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "servo-media-derive" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_derive" +proc-macro = true +path = "lib.rs" + +[dependencies] +syn = "2" +quote = "1" +proc-macro2 = "1" diff --git a/components/media/servo-media-derive/lib.rs b/components/media/servo-media-derive/lib.rs new file mode 100644 index 00000000000..84f8810f1be --- /dev/null +++ b/components/media/servo-media-derive/lib.rs @@ -0,0 +1,136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#![recursion_limit = "128"] + +extern crate proc_macro; +extern crate proc_macro2; +extern crate syn; +#[macro_use] +extern crate quote; + +use proc_macro::TokenStream; + +#[proc_macro_derive(AudioScheduledSourceNode)] +pub fn audio_scheduled_source_node(input: TokenStream) -> TokenStream { + let ast: syn::DeriveInput = syn::parse(input).unwrap(); + let r#gen = impl_audio_scheduled_source_node(&ast); + r#gen.into() +} + +fn impl_audio_scheduled_source_node(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { + let name = &ast.ident; + quote! { + impl #name { + fn should_play_at(&mut self, tick: Tick) -> ShouldPlay { + let start = if let Some(start) = self.start_at { + start + } else { + return ShouldPlay::No; + }; + + let frame_end = tick + Tick::FRAMES_PER_BLOCK; + if tick < start { + if frame_end < start { + ShouldPlay::No + } else { + let delta_start = start - tick; + if let Some(stop) = self.stop_at { + if stop <= start { + self.maybe_trigger_onended_callback(); + return ShouldPlay::No; + } + if stop > frame_end { + ShouldPlay::Between(delta_start, Tick::FRAMES_PER_BLOCK) + } else { + self.maybe_trigger_onended_callback(); + ShouldPlay::Between(delta_start, stop - tick) + } + } else { + ShouldPlay::Between(delta_start, Tick::FRAMES_PER_BLOCK) + } + } + } else { + let stop = if let Some(stop) = self.stop_at { + stop + } else { + return ShouldPlay::Between(Tick(0), Tick::FRAMES_PER_BLOCK); + }; + if stop > frame_end { + ShouldPlay::Between(Tick(0), Tick::FRAMES_PER_BLOCK) + } else if stop < tick { + self.maybe_trigger_onended_callback(); + ShouldPlay::No + } else { + self.maybe_trigger_onended_callback(); + ShouldPlay::Between(Tick(0), stop - tick) + } + } + } + + fn start(&mut self, tick: Tick) -> bool { + // We can only allow a single call to `start` and always before + // any `stop` calls. + if self.start_at.is_some() || self.stop_at.is_some() { + return false; + } + self.start_at = Some(tick); + true + } + + fn stop(&mut self, tick: Tick) -> bool { + // We can only allow calls to `stop` after `start` is called. + if self.start_at.is_none() { + return false; + } + // If `stop` is called again after already having been called, + // the last invocation will be the only one applied. + self.stop_at = Some(tick); + true + } + + fn maybe_trigger_onended_callback(&mut self) { + // We cannot have an end without a start. + if self.start_at.is_none() { + return; + } + if let Some(cb) = self.onended_callback.take() { + cb.0() + } + } + + fn handle_source_node_message(&mut self, message: AudioScheduledSourceNodeMessage, sample_rate: f32) { + match message { + AudioScheduledSourceNodeMessage::Start(when) => { + self.start(Tick::from_time(when, sample_rate)); + } + AudioScheduledSourceNodeMessage::Stop(when) => { + self.stop(Tick::from_time(when, sample_rate)); + } + AudioScheduledSourceNodeMessage::RegisterOnEndedCallback(callback) => { + self.onended_callback = Some(callback); + } + } + } + } + } +} + +#[proc_macro_derive(AudioNodeCommon)] +pub fn channel_info(input: TokenStream) -> TokenStream { + let ast: syn::DeriveInput = syn::parse(input).unwrap(); + let name = &ast.ident; + let r#gen = quote! { + impl crate::node::AudioNodeCommon for #name { + fn channel_info(&self) -> &crate::node::ChannelInfo { + &self.channel_info + } + + fn channel_info_mut(&mut self) -> &mut crate::node::ChannelInfo { + &mut self.channel_info + } + } + }; + r#gen.into() +} diff --git a/components/media/servo-media/Cargo.toml b/components/media/servo-media/Cargo.toml new file mode 100644 index 00000000000..afc50ab4583 --- /dev/null +++ b/components/media/servo-media/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "servo-media" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media" +path = "lib.rs" + +[dependencies] +once_cell = "1.18.0" + +[dependencies.servo-media-audio] +path = "../audio" + +[dependencies.servo-media-player] +path = "../player" + +[dependencies.servo-media-streams] +path = "../streams" + +[dependencies.servo-media-traits] +path = "../traits" + +[dependencies.servo-media-webrtc] +path = "../webrtc" diff --git a/components/media/servo-media/lib.rs b/components/media/servo-media/lib.rs new file mode 100644 index 00000000000..2f5f2ed6dd4 --- /dev/null +++ b/components/media/servo-media/lib.rs @@ -0,0 +1,123 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +pub extern crate servo_media_audio as audio; +pub extern crate servo_media_player as player; +pub extern crate servo_media_streams as streams; +pub extern crate servo_media_traits as traits; +pub extern crate servo_media_webrtc as webrtc; + +extern crate once_cell; + +use std::ops::Deref; +use std::sync::{Arc, Mutex}; +use std::thread; + +use audio::context::{AudioContext, AudioContextOptions}; +use audio::sink::AudioSinkError; +use once_cell::sync::OnceCell; +use player::audio::AudioRenderer; +use player::context::PlayerGLContext; +use player::ipc_channel::ipc::IpcSender; +use player::video::VideoFrameRenderer; +use player::{Player, PlayerEvent, StreamType}; +use streams::capture::MediaTrackConstraintSet; +use streams::device_monitor::MediaDeviceMonitor; +use streams::registry::MediaStreamId; +use streams::{MediaOutput, MediaSocket, MediaStreamType}; +pub use traits::*; +use webrtc::{WebRtcController, WebRtcSignaller}; + +pub struct ServoMedia(Box); + +static INSTANCE: OnceCell> = OnceCell::new(); + +pub trait BackendInit { + fn init() -> Box; +} + +pub trait BackendDeInit { + fn deinit(&self) {} +} + +pub trait Backend: Send + Sync { + fn create_player( + &self, + id: &ClientContextId, + stream_type: StreamType, + sender: IpcSender, + video_renderer: Option>>, + audio_renderer: Option>>, + gl_context: Box, + ) -> Arc>; + fn create_audiostream(&self) -> MediaStreamId; + fn create_videostream(&self) -> MediaStreamId; + fn create_stream_output(&self) -> Box; + fn create_stream_and_socket( + &self, + ty: MediaStreamType, + ) -> (Box, MediaStreamId); + fn create_audioinput_stream(&self, set: MediaTrackConstraintSet) -> Option; + fn create_videoinput_stream(&self, set: MediaTrackConstraintSet) -> Option; + fn create_audio_context( + &self, + id: &ClientContextId, + options: AudioContextOptions, + ) -> Result>, AudioSinkError>; + fn create_webrtc(&self, signaller: Box) -> WebRtcController; + fn can_play_type(&self, media_type: &str) -> SupportsMediaType; + fn set_capture_mocking(&self, _mock: bool) {} + /// Allow muting/unmuting the media instances associated with the given client context identifier. + /// Backend implementations are responsible for keeping a match between client contexts + /// and the media instances created for these contexts. + /// The client context identifier is currently an abstraction of Servo's PipelineId. + fn mute(&self, _id: &ClientContextId, _val: bool) {} + /// Allow suspending the activity of all media instances associated with the given client + /// context identifier. + /// Note that suspending does not involve releasing any resources, so media playback can + /// be restarted. + /// Backend implementations are responsible for keeping a match between client contexts + /// and the media instances created for these contexts. + /// The client context identifier is currently an abstraction of Servo's PipelineId. + fn suspend(&self, _id: &ClientContextId) {} + /// Allow resuming the activity of all the media instances associated with the given client + /// context identifier. + /// Backend implementations are responsible for keeping a match between client contexts + /// and the media instances created for these contexts. + /// The client context identifier is currently an abstraction of Servo's PipelineId. + fn resume(&self, _id: &ClientContextId) {} + + fn get_device_monitor(&self) -> Box; +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum SupportsMediaType { + Maybe, + No, + Probably, +} + +impl ServoMedia { + pub fn init() { + thread::spawn(|| INSTANCE.get_or_init(|| Arc::new(ServoMedia(B::init())))); + } + + pub fn init_with_backend(backend_factory: F) + where + F: Fn() -> Box + Send + 'static, + { + thread::spawn(move || INSTANCE.get_or_init(|| Arc::new(ServoMedia(backend_factory())))); + } + + pub fn get() -> Arc { + INSTANCE.wait().clone() + } +} + +impl Deref for ServoMedia { + type Target = dyn Backend + 'static; + fn deref(&self) -> &(dyn Backend + 'static) { + &*self.0 + } +} diff --git a/components/media/streams/Cargo.toml b/components/media/streams/Cargo.toml new file mode 100644 index 00000000000..8df58ee89b0 --- /dev/null +++ b/components/media/streams/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "servo-media-streams" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_streams" +path = "lib.rs" + +[dependencies] +uuid = { version = "1.4", features = ["v4"] } diff --git a/components/media/streams/capture.rs b/components/media/streams/capture.rs new file mode 100644 index 00000000000..a8baf0c0938 --- /dev/null +++ b/components/media/streams/capture.rs @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +pub struct ConstrainRange { + pub min: Option, + pub max: Option, + pub ideal: Option, +} + +pub enum ConstrainBool { + Ideal(bool), + Exact(bool), +} + +#[derive(Default)] +pub struct MediaTrackConstraintSet { + pub width: Option>, + pub height: Option>, + pub aspect: Option>, + pub frame_rate: Option>, + pub sample_rate: Option>, +} + +pub enum Constrain { + Value(T), + Range(ConstrainRange), +} diff --git a/components/media/streams/device_monitor.rs b/components/media/streams/device_monitor.rs new file mode 100644 index 00000000000..b98c0a3efd0 --- /dev/null +++ b/components/media/streams/device_monitor.rs @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#[derive(Clone, Copy, Debug)] +pub enum MediaDeviceKind { + AudioInput, + AudioOutput, + VideoInput, +} + +#[derive(Clone, Debug)] +pub struct MediaDeviceInfo { + pub device_id: String, + pub kind: MediaDeviceKind, + pub label: String, +} + +pub trait MediaDeviceMonitor { + fn enumerate_devices(&self) -> Option>; +} diff --git a/components/media/streams/lib.rs b/components/media/streams/lib.rs new file mode 100644 index 00000000000..9e0b692838e --- /dev/null +++ b/components/media/streams/lib.rs @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +pub mod capture; +pub mod device_monitor; +pub mod registry; + +use std::any::Any; + +pub use registry::*; + +pub trait MediaStream: Any + Send { + fn as_any(&self) -> &dyn Any; + fn as_mut_any(&mut self) -> &mut dyn Any; + fn set_id(&mut self, id: registry::MediaStreamId); + fn ty(&self) -> MediaStreamType; +} + +/// A MediaSocket is a way for a backend to represent a +/// yet-to-be-connected source side of a MediaStream +pub trait MediaSocket: Any + Send { + fn as_any(&self) -> &dyn Any; +} + +/// This isn't part of the webrtc spec; it's a leaky abstaction while media streams +/// are under development and example consumers need to be able to inspect them. +pub trait MediaOutput: Send { + fn add_stream(&mut self, stream: ®istry::MediaStreamId); +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum MediaStreamType { + Video, + Audio, +} diff --git a/components/media/streams/registry.rs b/components/media/streams/registry.rs new file mode 100644 index 00000000000..1f1a9cf0061 --- /dev/null +++ b/components/media/streams/registry.rs @@ -0,0 +1,42 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::collections::HashMap; +use std::sync::{Arc, LazyLock, Mutex}; + +use uuid::Uuid; + +use super::MediaStream; + +type RegisteredMediaStream = Arc>; + +static MEDIA_STREAMS_REGISTRY: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +#[derive(Clone, Copy, Default, Hash, Eq, PartialEq)] +pub struct MediaStreamId(Uuid); +impl MediaStreamId { + pub fn new() -> MediaStreamId { + Default::default() + } + + pub fn id(self) -> Uuid { + self.0 + } +} + +pub fn register_stream(stream: Arc>) -> MediaStreamId { + let id = MediaStreamId::new(); + stream.lock().unwrap().set_id(id); + MEDIA_STREAMS_REGISTRY.lock().unwrap().insert(id, stream); + id +} + +pub fn unregister_stream(stream: &MediaStreamId) { + MEDIA_STREAMS_REGISTRY.lock().unwrap().remove(stream); +} + +pub fn get_stream(stream: &MediaStreamId) -> Option>> { + MEDIA_STREAMS_REGISTRY.lock().unwrap().get(stream).cloned() +} diff --git a/components/media/traits/Cargo.toml b/components/media/traits/Cargo.toml new file mode 100644 index 00000000000..110319877d1 --- /dev/null +++ b/components/media/traits/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "servo-media-traits" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "servo_media_traits" +path = "lib.rs" diff --git a/components/media/traits/lib.rs b/components/media/traits/lib.rs new file mode 100644 index 00000000000..5985de17aa2 --- /dev/null +++ b/components/media/traits/lib.rs @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::num::NonZeroU32; +use std::sync::mpsc::Sender; +/// An ID for clients to track instances of Players and AudioContexts belonging to the same tab and mute them simultaneously. +/// Current tuple implementation matches one of Servo's BrowsingContextId. +#[derive(PartialEq, Eq, Hash, Debug, Clone, Copy)] +pub struct ClientContextId(u32, NonZeroU32); + +impl ClientContextId { + pub fn build(a: u32, b: u32) -> ClientContextId { + ClientContextId(a, NonZeroU32::new(b).unwrap()) + } +} + +#[derive(Debug)] +pub struct MediaInstanceError; + +impl std::fmt::Display for MediaInstanceError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "MediaInstanceError") + } +} + +impl std::error::Error for MediaInstanceError {} + +/// Common functionality for all high level media instances +/// These currently are WebAudio AudioContexts and Players. +pub trait MediaInstance: Send { + fn get_id(&self) -> usize; + fn mute(&self, val: bool) -> Result<(), MediaInstanceError>; + fn suspend(&self) -> Result<(), MediaInstanceError>; + fn resume(&self) -> Result<(), MediaInstanceError>; +} + +pub enum BackendMsg { + /// Message to notify about a media instance shutdown. + /// The given `usize` is the media instance ID. + Shutdown { + context: ClientContextId, + id: usize, + tx_ack: Sender<()>, + }, +} diff --git a/components/media/webrtc/Cargo.toml b/components/media/webrtc/Cargo.toml new file mode 100644 index 00000000000..a7e4b4a8a61 --- /dev/null +++ b/components/media/webrtc/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "servo-media-webrtc" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +path = "lib.rs" + +[dependencies] +log = "0.4" +uuid = { version = "1.4", features = ["v4"] } + +[dependencies.servo-media-streams] +path = "../streams" diff --git a/components/media/webrtc/datachannel.rs b/components/media/webrtc/datachannel.rs new file mode 100644 index 00000000000..2075061c90d --- /dev/null +++ b/components/media/webrtc/datachannel.rs @@ -0,0 +1,59 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use uuid::Uuid; + +use crate::WebRtcError; + +pub type DataChannelId = usize; + +#[derive(Debug)] +pub enum DataChannelMessage { + Text(String), + Binary(Vec), +} + +#[derive(Debug)] +pub enum DataChannelState { + Connecting, + Open, + Closing, + Closed, + __Unknown(i32), +} + +pub enum DataChannelEvent { + NewChannel, + Open, + Close, + Error(WebRtcError), + OnMessage(DataChannelMessage), + StateChange(DataChannelState), +} + +// https://www.w3.org/TR/webrtc/#dom-rtcdatachannelinit +// plus `label` +pub struct DataChannelInit { + pub label: String, + pub ordered: bool, + pub max_packet_life_time: Option, + pub max_retransmits: Option, + pub protocol: String, + pub negotiated: bool, + pub id: Option, +} + +impl Default for DataChannelInit { + fn default() -> DataChannelInit { + DataChannelInit { + label: Uuid::new_v4().to_string(), + ordered: true, + max_packet_life_time: None, + max_retransmits: None, + protocol: String::new(), + negotiated: false, + id: None, + } + } +} diff --git a/components/media/webrtc/lib.rs b/components/media/webrtc/lib.rs new file mode 100644 index 00000000000..2e9f60d77ce --- /dev/null +++ b/components/media/webrtc/lib.rs @@ -0,0 +1,201 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +extern crate log; +extern crate servo_media_streams; +extern crate uuid; + +use std::fmt::Display; +use std::str::FromStr; + +use servo_media_streams::MediaStreamType; +use servo_media_streams::registry::MediaStreamId; + +pub mod datachannel; +pub mod thread; + +pub use datachannel::{ + DataChannelEvent, DataChannelId, DataChannelInit, DataChannelMessage, DataChannelState, +}; +pub use thread::WebRtcController; + +#[derive(Debug)] +pub enum WebRtcError { + Backend(String), +} + +pub type WebRtcResult = Result<(), WebRtcError>; +pub type WebRtcDataChannelResult = Result; + +impl From for WebRtcError { + fn from(x: T) -> Self { + WebRtcError::Backend(x.to_string()) + } +} + +/// This trait is implemented by backends and should never be used directly by +/// the client. Use WebRtcController instead +pub trait WebRtcControllerBackend: Send { + fn configure(&mut self, stun_server: &str, policy: BundlePolicy) -> WebRtcResult; + fn set_remote_description( + &mut self, + _: SessionDescription, + cb: Box, + ) -> WebRtcResult; + fn set_local_description( + &mut self, + _: SessionDescription, + cb: Box, + ) -> WebRtcResult; + fn add_ice_candidate(&mut self, candidate: IceCandidate) -> WebRtcResult; + fn create_offer( + &mut self, + cb: Box, + ) -> WebRtcResult; + fn create_answer( + &mut self, + cb: Box, + ) -> WebRtcResult; + fn add_stream(&mut self, stream: &MediaStreamId) -> WebRtcResult; + + fn create_data_channel(&mut self, init: &DataChannelInit) -> WebRtcDataChannelResult; + fn close_data_channel(&mut self, channel: &DataChannelId) -> WebRtcResult; + fn send_data_channel_message( + &mut self, + channel: &DataChannelId, + message: &DataChannelMessage, + ) -> WebRtcResult; + + fn internal_event(&mut self, event: thread::InternalEvent) -> WebRtcResult; + fn quit(&mut self); +} + +pub trait WebRtcSignaller: Send { + fn on_ice_candidate(&self, controller: &WebRtcController, candidate: IceCandidate); + fn on_negotiation_needed(&self, controller: &WebRtcController); + fn close(&self); + fn on_add_stream(&self, stream: &MediaStreamId, ty: MediaStreamType); + + fn update_signaling_state(&self, _: SignalingState) {} + fn update_gathering_state(&self, _: GatheringState) {} + fn update_ice_connection_state(&self, _: IceConnectionState) {} + + fn on_data_channel_event(&self, _: DataChannelId, _: DataChannelEvent, _: &WebRtcController) {} +} + +pub trait WebRtcBackend { + type Controller: WebRtcControllerBackend + 'static; + + fn construct_webrtc_controller( + signaller: Box, + thread: WebRtcController, + ) -> Self::Controller; +} + +/// +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq)] +pub enum SdpType { + Answer, + Offer, + Pranswer, + Rollback, +} + +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq)] +pub enum DescriptionType { + Local, + Remote, +} + +impl SdpType { + pub fn as_str(self) -> &'static str { + match self { + SdpType::Answer => "answer", + SdpType::Offer => "offer", + SdpType::Pranswer => "pranswer", + SdpType::Rollback => "rollback", + } + } +} + +impl FromStr for SdpType { + type Err = (); + fn from_str(s: &str) -> Result { + Ok(match s { + "answer" => SdpType::Answer, + "offer" => SdpType::Offer, + "pranswer" => SdpType::Pranswer, + "rollback" => SdpType::Rollback, + _ => return Err(()), + }) + } +} + +/// +/// +/// +#[derive(Clone, Hash, Debug, PartialEq, Eq)] +pub struct SessionDescription { + pub type_: SdpType, + pub sdp: String, +} + +/// +/// +/// +#[derive(Clone, Hash, Debug, PartialEq, Eq)] +pub struct IceCandidate { + pub sdp_mline_index: u32, + pub candidate: String, + // XXXManishearth this is missing a bunch +} + +/// +#[derive(Clone, Copy, Hash, Debug, PartialEq, Eq)] +pub enum BundlePolicy { + Balanced, + MaxCompat, + MaxBundle, +} + +impl BundlePolicy { + pub fn as_str(self) -> &'static str { + match self { + BundlePolicy::Balanced => "balanced", + BundlePolicy::MaxCompat => "max-compat", + BundlePolicy::MaxBundle => "max-bundle", + } + } +} + +/// +#[derive(Clone, Copy, Hash, Debug, PartialEq, Eq)] +pub enum SignalingState { + Stable, + HaveLocalOffer, + HaveRemoteOffer, + HaveLocalPranswer, + HaveRemotePranswer, + Closed, +} + +/// +#[derive(Clone, Copy, Hash, Debug, PartialEq, Eq)] +pub enum GatheringState { + New, + Gathering, + Complete, +} + +/// +#[derive(Clone, Copy, Hash, Debug, PartialEq, Eq)] +pub enum IceConnectionState { + New, + Checking, + Connected, + Completed, + Disconnected, + Failed, + Closed, +} diff --git a/components/media/webrtc/thread.rs b/components/media/webrtc/thread.rs new file mode 100644 index 00000000000..288fa429958 --- /dev/null +++ b/components/media/webrtc/thread.rs @@ -0,0 +1,175 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::sync::mpsc::{Sender, channel}; +use std::thread; + +use log::error; +use servo_media_streams::MediaStreamType; + +use crate::datachannel::{DataChannelEvent, DataChannelId, DataChannelInit, DataChannelMessage}; +use crate::{ + BundlePolicy, DescriptionType, IceCandidate, MediaStreamId, SdpType, SessionDescription, + WebRtcBackend, WebRtcControllerBackend, WebRtcSignaller, +}; + +#[derive(Clone)] +/// Entry point for all client webrtc interactions. +pub struct WebRtcController { + sender: Sender, +} + +impl WebRtcController { + pub fn new(signaller: Box) -> Self { + let (sender, receiver) = channel(); + + let t = WebRtcController { sender }; + + let mut controller = T::construct_webrtc_controller(signaller, t.clone()); + + thread::spawn(move || { + while let Ok(event) = receiver.recv() { + if !handle_rtc_event(&mut controller, event) { + // shut down event loop + break; + } + } + }); + + t + } + pub fn configure(&self, stun_server: String, policy: BundlePolicy) { + let _ = self + .sender + .send(RtcThreadEvent::ConfigureStun(stun_server, policy)); + } + pub fn set_remote_description( + &self, + desc: SessionDescription, + cb: Box, + ) { + let _ = self + .sender + .send(RtcThreadEvent::SetRemoteDescription(desc, cb)); + } + pub fn set_local_description( + &self, + desc: SessionDescription, + cb: Box, + ) { + let _ = self + .sender + .send(RtcThreadEvent::SetLocalDescription(desc, cb)); + } + pub fn add_ice_candidate(&self, candidate: IceCandidate) { + let _ = self.sender.send(RtcThreadEvent::AddIceCandidate(candidate)); + } + pub fn create_offer(&self, cb: Box) { + let _ = self.sender.send(RtcThreadEvent::CreateOffer(cb)); + } + pub fn create_answer(&self, cb: Box) { + let _ = self.sender.send(RtcThreadEvent::CreateAnswer(cb)); + } + pub fn add_stream(&self, stream: &MediaStreamId) { + let _ = self.sender.send(RtcThreadEvent::AddStream(*stream)); + } + pub fn create_data_channel(&self, init: DataChannelInit) -> Option { + let (sender, receiver) = channel(); + let _ = self + .sender + .send(RtcThreadEvent::CreateDataChannel(init, sender)); + receiver.recv().unwrap() + } + pub fn send_data_channel_message(&self, id: &DataChannelId, message: DataChannelMessage) { + let _ = self + .sender + .send(RtcThreadEvent::SendDataChannelMessage(*id, message)); + } + pub fn close_data_channel(&self, id: &DataChannelId) { + let _ = self.sender.send(RtcThreadEvent::CloseDataChannel(*id)); + } + + /// This should not be invoked by clients + pub fn internal_event(&self, event: InternalEvent) { + let _ = self.sender.send(RtcThreadEvent::InternalEvent(event)); + } + + pub fn quit(&self) { + let _ = self.sender.send(RtcThreadEvent::Quit); + } +} + +pub enum RtcThreadEvent { + ConfigureStun(String, BundlePolicy), + SetRemoteDescription(SessionDescription, Box), + SetLocalDescription(SessionDescription, Box), + AddIceCandidate(IceCandidate), + CreateOffer(Box), + CreateAnswer(Box), + AddStream(MediaStreamId), + CreateDataChannel(DataChannelInit, Sender>), + CloseDataChannel(DataChannelId), + SendDataChannelMessage(DataChannelId, DataChannelMessage), + InternalEvent(InternalEvent), + Quit, +} + +/// To allow everything to occur on the event loop, +/// the backend may need to send signals to itself +/// +/// This is a somewhat leaky abstraction, but we don't +/// plan on having too many backends anyway +pub enum InternalEvent { + OnNegotiationNeeded, + OnIceCandidate(IceCandidate), + OnAddStream(MediaStreamId, MediaStreamType), + OnDataChannelEvent(DataChannelId, DataChannelEvent), + DescriptionAdded( + Box, + DescriptionType, + SdpType, + /* remote offer generation */ u32, + ), + UpdateSignalingState, + UpdateGatheringState, + UpdateIceConnectionState, +} + +pub fn handle_rtc_event( + controller: &mut dyn WebRtcControllerBackend, + event: RtcThreadEvent, +) -> bool { + let result = match event { + RtcThreadEvent::ConfigureStun(server, policy) => controller.configure(&server, policy), + RtcThreadEvent::SetRemoteDescription(desc, cb) => { + controller.set_remote_description(desc, cb) + }, + RtcThreadEvent::SetLocalDescription(desc, cb) => controller.set_local_description(desc, cb), + RtcThreadEvent::AddIceCandidate(candidate) => controller.add_ice_candidate(candidate), + RtcThreadEvent::CreateOffer(cb) => controller.create_offer(cb), + RtcThreadEvent::CreateAnswer(cb) => controller.create_answer(cb), + RtcThreadEvent::AddStream(media) => controller.add_stream(&media), + RtcThreadEvent::CreateDataChannel(init, sender) => controller + .create_data_channel(&init) + .map(|id| { + let _ = sender.send(Some(id)); + }) + .inspect_err(|_| { + let _ = sender.send(None); + }), + RtcThreadEvent::CloseDataChannel(id) => controller.close_data_channel(&id), + RtcThreadEvent::SendDataChannelMessage(id, message) => { + controller.send_data_channel_message(&id, &message) + }, + RtcThreadEvent::InternalEvent(e) => controller.internal_event(e), + RtcThreadEvent::Quit => { + controller.quit(); + return false; + }, + }; + if let Err(e) = result { + error!("WebRTC backend encountered error: {:?}", e); + } + true +} diff --git a/components/paint/Cargo.toml b/components/paint/Cargo.toml index 16b611b1dde..ffc7b9704f9 100644 --- a/components/paint/Cargo.toml +++ b/components/paint/Cargo.toml @@ -34,7 +34,7 @@ gleam = { workspace = true } image = { workspace = true } ipc-channel = { workspace = true } log = { workspace = true } -media = { path = "../media" } +media = { path = "../media/media-thread" } malloc_size_of = { workspace = true } paint_api = { workspace = true } profile_traits = { workspace = true } diff --git a/components/script/Cargo.toml b/components/script/Cargo.toml index 413a9db0a13..4849a282fc5 100644 --- a/components/script/Cargo.toml +++ b/components/script/Cargo.toml @@ -95,7 +95,7 @@ log = { workspace = true } malloc_size_of = { workspace = true } malloc_size_of_derive = { workspace = true } markup5ever = { workspace = true } -media = { path = "../media" } +media = { path = "../media/media-thread" } metrics = { path = "../metrics" } mime = { workspace = true } mime-multipart-hyper1 = { workspace = true } diff --git a/components/script/dom/audio/audiocontext.rs b/components/script/dom/audio/audiocontext.rs index 2a76ef6040f..90f66de15d1 100644 --- a/components/script/dom/audio/audiocontext.rs +++ b/components/script/dom/audio/audiocontext.rs @@ -159,7 +159,7 @@ impl AudioContextMethods for AudioContext { // Steps 4 and 5. let trusted_promise = TrustedPromise::new(promise.clone()); match self.context.audio_context_impl().lock().unwrap().suspend() { - Ok(_) => { + Some(_) => { let base_context = Trusted::new(&self.context); let context = Trusted::new(self); self.global().task_manager().dom_manipulation_task_source().queue( @@ -178,7 +178,7 @@ impl AudioContextMethods for AudioContext { }) ); }, - Err(_) => { + None => { // The spec does not define the error case and `suspend` should // never fail, but we handle the case here for completion. self.global() @@ -215,7 +215,7 @@ impl AudioContextMethods for AudioContext { // Steps 4 and 5. let trusted_promise = TrustedPromise::new(promise.clone()); match self.context.audio_context_impl().lock().unwrap().close() { - Ok(_) => { + Some(_) => { let base_context = Trusted::new(&self.context); let context = Trusted::new(self); self.global().task_manager().dom_manipulation_task_source().queue( @@ -234,7 +234,7 @@ impl AudioContextMethods for AudioContext { }) ); }, - Err(_) => { + None => { // The spec does not define the error case and `suspend` should // never fail, but we handle the case here for completion. self.global() diff --git a/components/script/dom/audio/audionode.rs b/components/script/dom/audio/audionode.rs index 90e07254aab..eec913fa170 100644 --- a/components/script/dom/audio/audionode.rs +++ b/components/script/dom/audio/audionode.rs @@ -65,21 +65,19 @@ impl AudioNode { interpretation: options.interpretation.convert(), context_channel_count: context.channel_count() as u8, }; - let node_id = match context + let node_id = context .audio_context_impl() .lock() .unwrap() - .create_node(node_type, ch) - { - Ok(node_id) => Some(node_id), - Err(_) => { - // Follow Chromuim and Gecko, we just warn and create an inert AudioNode. - const MESSAGE: &str = "Failed to create an AudioNode backend. The constructed AudioNode will be inert."; - warn!("{MESSAGE}"); - Console::internal_warn(&context.global(), DOMString::from(MESSAGE)); - None - }, - }; + .create_node(node_type, ch); + + if node_id.is_none() { + // Follow Chromuim and Gecko, we just warn and create an inert AudioNode. + const MESSAGE: &str = + "Failed to create an AudioNode backend. The constructed AudioNode will be inert."; + warn!("{MESSAGE}"); + Console::internal_warn(&context.global(), DOMString::from(MESSAGE)); + } Ok(AudioNode::new_inherited_for_id( node_id, diff --git a/components/script/dom/audio/baseaudiocontext.rs b/components/script/dom/audio/baseaudiocontext.rs index 8d17fd18057..ad705cff950 100644 --- a/components/script/dom/audio/baseaudiocontext.rs +++ b/components/script/dom/audio/baseaudiocontext.rs @@ -16,7 +16,7 @@ use servo_media::audio::context::{ AudioContext, AudioContextOptions, OfflineAudioContextOptions, ProcessingState, RealTimeAudioContextOptions, }; -use servo_media::audio::decoder::AudioDecoderCallbacks; +use servo_media::audio::decoder::AudioDecoderCallbacksBuilder; use servo_media::audio::graph::NodeId; use servo_media::{ClientContextId, ServoMedia}; use uuid::Uuid; @@ -231,7 +231,7 @@ impl BaseAudioContext { // Set the rendering thread state to 'running' and start // rendering the audio graph. match self.audio_context_impl.lock().unwrap().resume() { - Ok(()) => { + Some(()) => { self.take_pending_resume_promises(Ok(())); self.global().task_manager().dom_manipulation_task_source().queue( task!(resume_success: move || { @@ -248,7 +248,7 @@ impl BaseAudioContext { }) ); }, - Err(()) => { + None => { self.take_pending_resume_promises(Err(Error::Type( "Something went wrong".to_owned(), ))); @@ -507,7 +507,7 @@ impl BaseAudioContextMethods for BaseAudioContext { .dom_manipulation_task_source() .to_sendable(); let task_source_clone = task_source.clone(); - let callbacks = AudioDecoderCallbacks::new() + let callbacks = AudioDecoderCallbacksBuilder::default() .ready(move |channel_count| { decoded_audio .lock() diff --git a/components/script/dom/audio/offlineaudiocontext.rs b/components/script/dom/audio/offlineaudiocontext.rs index b3efd365538..8d29a42b365 100644 --- a/components/script/dom/audio/offlineaudiocontext.rs +++ b/components/script/dom/audio/offlineaudiocontext.rs @@ -219,7 +219,7 @@ impl OfflineAudioContextMethods for OfflineAudioContext { .lock() .unwrap() .resume() - .is_err() + .is_none() { promise.reject_error( Error::Type("Could not start offline rendering".to_owned()), diff --git a/components/script/dom/media/mediadevices.rs b/components/script/dom/media/mediadevices.rs index 6b8d1f1eec6..6e02d0a5cfd 100644 --- a/components/script/dom/media/mediadevices.rs +++ b/components/script/dom/media/mediadevices.rs @@ -88,23 +88,25 @@ impl MediaDevicesMethods for MediaDevices { // Step 2.5 let media = ServoMedia::get(); let device_monitor = media.get_device_monitor(); - let result_list = match device_monitor.enumerate_devices() { - Ok(devices) => devices - .iter() - .map(|device| { - // XXX The media backend has no way to group devices yet. - MediaDeviceInfo::new( - &self.global(), - &device.device_id, - device.kind.convert(), - &device.label, - "", - can_gc, - ) - }) - .collect(), - Err(_) => Vec::new(), - }; + let result_list = device_monitor + .enumerate_devices() + .map(|devices| { + devices + .iter() + .map(|device| { + // XXX The media backend has no way to group devices yet. + MediaDeviceInfo::new( + &self.global(), + &device.device_id, + device.kind.convert(), + &device.label, + "", + can_gc, + ) + }) + .collect::>() + }) + .unwrap_or_default(); p.resolve_native(&result_list, can_gc); diff --git a/components/servo/Cargo.toml b/components/servo/Cargo.toml index a19e1650fa6..0ef9b23c696 100644 --- a/components/servo/Cargo.toml +++ b/components/servo/Cargo.toml @@ -97,7 +97,7 @@ keyboard-types = { workspace = true } layout = { path = "../layout" } layout_api = { workspace = true } log = { workspace = true } -media = { path = "../media" } +media = { path = "../media/media-thread" } mozangle = { workspace = true } net = { path = "../net" } net_traits = { workspace = true } diff --git a/components/shared/script/Cargo.toml b/components/shared/script/Cargo.toml index 81306b5bdec..3aa42a5b88f 100644 --- a/components/shared/script/Cargo.toml +++ b/components/shared/script/Cargo.toml @@ -32,7 +32,7 @@ keyboard-types = { workspace = true } log = { workspace = true } malloc_size_of = { workspace = true } malloc_size_of_derive = { workspace = true } -media = { path = "../../media" } +media = { path = "../../media/media-thread" } net_traits = { workspace = true } paint_api = { workspace = true } pixels = { path = "../../pixels" } diff --git a/docs/media/avplayback.md b/docs/media/avplayback.md new file mode 100644 index 00000000000..90e5235e394 --- /dev/null +++ b/docs/media/avplayback.md @@ -0,0 +1,104 @@ +# A/V Playback through HTMLMediaElement + +Media elements are used to present audio and/or video data to the user. The [HTMLMediaElement](https://html.spec.whatwg.org/multipage/media.html#htmlmediaelement) interface adds to [HTMLElement](https://html.spec.whatwg.org/multipage/dom.html#htmlelement) the properties and methods needed to support basic media-related capabilities that are common to audio and video. The [HTMLVideoElement](https://html.spec.whatwg.org/multipage/media.html#htmlvideoelement) (`