mirror of
https://github.com/LadybirdBrowser/ladybird
synced 2026-04-26 01:35:08 +02:00
08faa83340
This state will indicate to the media element that it's not guaranteed to have a frame yet, for the purposes of determining the ready state. JavaScript should be sure that video elements with a ready state of HAVE_CURRENT_DATA or greater represent the current video frame already. To allow the state to be exited if audio is disabled, audio tracks are now only added to the buffering set on enable if the audio sink exists, since without the sink starting the data provider, it will never be removed. This is a step towards making video ref tests.
453 lines
16 KiB
C++
453 lines
16 KiB
C++
/*
|
|
* Copyright (c) 2022-2026, Gregory Bertilson <gregory@ladybird.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#include <LibMedia/Containers/Matroska/MatroskaDemuxer.h>
|
|
#include <LibMedia/Demuxer.h>
|
|
#include <LibMedia/FFmpeg/FFmpegDemuxer.h>
|
|
#include <LibMedia/PlaybackStates/StartingStateHandler.h>
|
|
#include <LibMedia/Providers/AudioDataProvider.h>
|
|
#include <LibMedia/Providers/GenericTimeProvider.h>
|
|
#include <LibMedia/Providers/VideoDataProvider.h>
|
|
#include <LibMedia/Providers/WrapperTimeProvider.h>
|
|
#include <LibMedia/Sinks/AudioMixingSink.h>
|
|
#include <LibMedia/Sinks/DisplayingVideoSink.h>
|
|
#include <LibMedia/Track.h>
|
|
#include <LibThreading/Thread.h>
|
|
#include <LibThreading/ThreadPool.h>
|
|
|
|
#include "PlaybackManager.h"
|
|
|
|
namespace Media {
|
|
|
|
DecoderErrorOr<NonnullRefPtr<Demuxer>> PlaybackManager::create_demuxer_for_stream(NonnullRefPtr<MediaStream> const& stream)
|
|
{
|
|
if (Matroska::Reader::is_matroska_or_webm(stream->create_cursor()))
|
|
return Matroska::MatroskaDemuxer::from_stream(stream);
|
|
return FFmpeg::FFmpegDemuxer::from_stream(stream);
|
|
}
|
|
|
|
DecoderErrorOr<void> PlaybackManager::prepare_playback_from_demuxer(WeakPlaybackManager const& self, NonnullRefPtr<Demuxer> const& demuxer, NonnullRefPtr<Core::WeakEventLoopReference> const& main_thread_event_loop_reference)
|
|
{
|
|
// Create the video tracks and their data providers.
|
|
auto all_video_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Video));
|
|
|
|
auto supported_video_tracks = VideoTracks();
|
|
auto supported_video_track_datas = VideoTrackDatas();
|
|
supported_video_tracks.ensure_capacity(all_video_tracks.size());
|
|
supported_video_track_datas.ensure_capacity(all_video_tracks.size());
|
|
for (auto const& track : all_video_tracks) {
|
|
auto video_data_provider_result = VideoDataProvider::try_create(main_thread_event_loop_reference, demuxer, track);
|
|
if (video_data_provider_result.is_error())
|
|
continue;
|
|
supported_video_tracks.append(track);
|
|
supported_video_track_datas.empend(VideoTrackData(track, video_data_provider_result.release_value(), nullptr));
|
|
}
|
|
supported_video_tracks.shrink_to_fit();
|
|
supported_video_track_datas.shrink_to_fit();
|
|
|
|
// Create all the audio tracks, their data providers, and the audio output.
|
|
auto all_audio_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Audio));
|
|
|
|
auto supported_audio_tracks = AudioTracks();
|
|
auto supported_audio_track_datas = AudioTrackDatas();
|
|
supported_audio_tracks.ensure_capacity(all_audio_tracks.size());
|
|
supported_audio_track_datas.ensure_capacity(all_audio_tracks.size());
|
|
for (auto const& track : all_audio_tracks) {
|
|
auto audio_data_provider_result = AudioDataProvider::try_create(main_thread_event_loop_reference, demuxer, track);
|
|
if (audio_data_provider_result.is_error())
|
|
continue;
|
|
auto audio_data_provider = audio_data_provider_result.release_value();
|
|
supported_audio_tracks.append(track);
|
|
supported_audio_track_datas.empend(AudioTrackData(track, move(audio_data_provider)));
|
|
}
|
|
supported_audio_tracks.shrink_to_fit();
|
|
supported_audio_track_datas.shrink_to_fit();
|
|
|
|
if (supported_video_tracks.is_empty() && supported_audio_tracks.is_empty())
|
|
return DecoderError::with_description(DecoderErrorCategory::NotImplemented, "No supported video or audio tracks found"sv);
|
|
|
|
auto preferred_video_track = demuxer->get_preferred_track_for_type(TrackType::Video).value_or({});
|
|
if (preferred_video_track.has_value() && !supported_video_tracks.contains_slow(*preferred_video_track))
|
|
preferred_video_track = {};
|
|
auto preferred_audio_track = demuxer->get_preferred_track_for_type(TrackType::Audio).value_or({});
|
|
if (preferred_audio_track.has_value() && !supported_audio_tracks.contains_slow(*preferred_audio_track))
|
|
preferred_audio_track = {};
|
|
|
|
auto duration = demuxer->total_duration().value_or(AK::Duration::zero());
|
|
|
|
auto main_thread_event_loop = main_thread_event_loop_reference->take();
|
|
main_thread_event_loop->deferred_invoke([self, video_tracks = move(supported_video_tracks), video_track_datas = move(supported_video_track_datas), preferred_video_track, audio_tracks = move(supported_audio_tracks), audio_track_datas = move(supported_audio_track_datas), preferred_audio_track, duration] mutable {
|
|
if (!self)
|
|
return;
|
|
|
|
for (auto const& existing_track : self->m_video_tracks) {
|
|
if (video_tracks.contains_slow(existing_track)) {
|
|
self->on_unsupported_format_error(DecoderError::with_description(DecoderErrorCategory::Invalid, "Duplicate video track found"sv));
|
|
return;
|
|
}
|
|
}
|
|
for (auto const& existing_track : self->m_audio_tracks) {
|
|
if (audio_tracks.contains_slow(existing_track)) {
|
|
self->on_unsupported_format_error(DecoderError::with_description(DecoderErrorCategory::Invalid, "Duplicate audio track found"sv));
|
|
return;
|
|
}
|
|
}
|
|
|
|
auto first_new_video_index = self->m_video_tracks.size();
|
|
auto first_new_audio_index = self->m_audio_tracks.size();
|
|
|
|
self->m_video_tracks.extend(move(video_tracks));
|
|
self->m_video_track_datas.extend(move(video_track_datas));
|
|
self->m_audio_tracks.extend(move(audio_tracks));
|
|
self->m_audio_track_datas.extend(move(audio_track_datas));
|
|
|
|
if (!self->m_preferred_video_track.has_value())
|
|
self->m_preferred_video_track = preferred_video_track;
|
|
if (!self->m_preferred_audio_track.has_value())
|
|
self->m_preferred_audio_track = preferred_audio_track;
|
|
|
|
self->check_for_duration_change(duration);
|
|
|
|
self->set_up_data_providers();
|
|
|
|
if (!self->m_audio_output_disabled && !self->m_audio_sink && !self->m_audio_tracks.is_empty()) {
|
|
self->m_audio_sink = MUST(AudioMixingSink::try_create());
|
|
self->set_time_provider(make_ref_counted<WrapperTimeProvider<AudioMixingSink>>(*self->m_audio_sink));
|
|
self->m_audio_sink->on_audio_output_error = [self](Error&& error) {
|
|
if (!self)
|
|
return;
|
|
dbgln("Audio output initialization failed with error: {}", error);
|
|
self->disable_audio();
|
|
};
|
|
self->m_audio_sink->on_start_buffering = [self](Track const& track) {
|
|
if (!self)
|
|
return;
|
|
self->track_started_buffering(track);
|
|
};
|
|
}
|
|
|
|
if (self->on_track_added) {
|
|
for (size_t i = first_new_audio_index; i < self->m_audio_tracks.size(); i++)
|
|
self->on_track_added(self->m_audio_tracks[i]);
|
|
for (size_t i = first_new_video_index; i < self->m_video_tracks.size(); i++)
|
|
self->on_track_added(self->m_video_tracks[i]);
|
|
}
|
|
|
|
if (self->on_metadata_parsed)
|
|
self->on_metadata_parsed();
|
|
});
|
|
|
|
return {};
|
|
}
|
|
|
|
NonnullOwnPtr<PlaybackManager> PlaybackManager::create()
|
|
{
|
|
auto playback_manager = adopt_own(*new (nothrow) PlaybackManager());
|
|
playback_manager->m_handler = make<StartingStateHandler>(*playback_manager);
|
|
playback_manager->m_handler->on_enter();
|
|
return playback_manager;
|
|
}
|
|
|
|
PlaybackManager::PlaybackManager()
|
|
: m_weak_link(make_ref_counted<WeakPlaybackManagerLink>(*this))
|
|
, m_time_provider(make_ref_counted<GenericTimeProvider>())
|
|
{
|
|
}
|
|
|
|
PlaybackManager::~PlaybackManager()
|
|
{
|
|
m_weak_link->revoke({});
|
|
}
|
|
|
|
static void handle_media_init_error(WeakPlaybackManager self, NonnullRefPtr<Core::WeakEventLoopReference> const& main_thread_event_loop_reference, DecoderError error)
|
|
{
|
|
auto main_thread_event_loop = main_thread_event_loop_reference->take();
|
|
if (!main_thread_event_loop)
|
|
return;
|
|
main_thread_event_loop->deferred_invoke([self = move(self), error = move(error)] mutable {
|
|
if (!self)
|
|
return;
|
|
if (self->on_unsupported_format_error)
|
|
self->on_unsupported_format_error(move(error));
|
|
});
|
|
}
|
|
|
|
void PlaybackManager::add_media_source(NonnullRefPtr<MediaStream> const& stream)
|
|
{
|
|
auto self = weak();
|
|
auto main_thread_event_loop_reference = Core::EventLoop::current_weak();
|
|
|
|
Threading::ThreadPool::the().submit([self = move(self), stream, main_thread_event_loop_reference = move(main_thread_event_loop_reference)] mutable {
|
|
auto demuxer_or_error = create_demuxer_for_stream(stream);
|
|
if (demuxer_or_error.is_error()) {
|
|
handle_media_init_error(move(self), move(main_thread_event_loop_reference), demuxer_or_error.release_error());
|
|
return;
|
|
}
|
|
|
|
auto maybe_error = prepare_playback_from_demuxer(self, demuxer_or_error.release_value(), main_thread_event_loop_reference);
|
|
if (maybe_error.is_error())
|
|
handle_media_init_error(move(self), move(main_thread_event_loop_reference), maybe_error.release_error());
|
|
});
|
|
}
|
|
|
|
void PlaybackManager::add_media_source(NonnullRefPtr<Demuxer> const& demuxer)
|
|
{
|
|
auto self = weak();
|
|
auto main_thread_event_loop_reference = Core::EventLoop::current_weak();
|
|
|
|
Threading::ThreadPool::the().submit([self = move(self), demuxer, main_thread_event_loop_reference = move(main_thread_event_loop_reference)] mutable {
|
|
auto maybe_error = prepare_playback_from_demuxer(self, demuxer, main_thread_event_loop_reference);
|
|
if (maybe_error.is_error())
|
|
handle_media_init_error(move(self), move(main_thread_event_loop_reference), maybe_error.release_error());
|
|
});
|
|
}
|
|
|
|
WeakPlaybackManager PlaybackManager::weak()
|
|
{
|
|
return WeakPlaybackManager(m_weak_link);
|
|
}
|
|
|
|
void PlaybackManager::set_up_data_providers()
|
|
{
|
|
for (auto const& video_track_data : m_video_track_datas) {
|
|
auto track = video_track_data.track;
|
|
video_track_data.provider->set_error_handler([self = weak(), track](DecoderError&& error) {
|
|
if (!self)
|
|
return;
|
|
if (error.category() == DecoderErrorCategory::EndOfStream) {
|
|
self->track_stopped_buffering(track);
|
|
return;
|
|
}
|
|
self->dispatch_error(move(error));
|
|
});
|
|
video_track_data.provider->set_duration_change_handler([self = weak()](AK::Duration time) {
|
|
if (!self)
|
|
return;
|
|
self->check_for_duration_change(time);
|
|
});
|
|
video_track_data.provider->set_frames_queue_is_full_handler([self = weak(), track] {
|
|
if (!self)
|
|
return;
|
|
self->track_stopped_buffering(track);
|
|
});
|
|
}
|
|
|
|
for (auto const& audio_track_data : m_audio_track_datas) {
|
|
auto track = audio_track_data.track;
|
|
audio_track_data.provider->set_error_handler([self = weak(), track](DecoderError&& error) {
|
|
if (!self)
|
|
return;
|
|
if (error.category() == DecoderErrorCategory::EndOfStream) {
|
|
self->track_stopped_buffering(track);
|
|
return;
|
|
}
|
|
self->dispatch_error(move(error));
|
|
});
|
|
audio_track_data.provider->set_duration_change_handler([self = weak()](AK::Duration time) {
|
|
if (!self)
|
|
return;
|
|
self->check_for_duration_change(time);
|
|
});
|
|
audio_track_data.provider->set_queue_is_full_handler([self = weak(), track] {
|
|
if (!self)
|
|
return;
|
|
self->track_stopped_buffering(track);
|
|
});
|
|
}
|
|
}
|
|
|
|
void PlaybackManager::track_started_buffering(Track const& track)
|
|
{
|
|
m_tracks_still_buffering.set(track);
|
|
m_handler->enter_buffering();
|
|
}
|
|
|
|
void PlaybackManager::track_stopped_buffering(Track const& track)
|
|
{
|
|
m_tracks_still_buffering.remove(track);
|
|
if (m_tracks_still_buffering.is_empty())
|
|
m_handler->exit_buffering();
|
|
}
|
|
|
|
void PlaybackManager::check_for_duration_change(AK::Duration duration)
|
|
{
|
|
if (m_duration >= duration)
|
|
return;
|
|
m_duration = duration;
|
|
if (on_duration_change)
|
|
on_duration_change(m_duration);
|
|
}
|
|
|
|
void PlaybackManager::dispatch_error(DecoderError&& error)
|
|
{
|
|
VERIFY(error.category() != DecoderErrorCategory::EndOfStream);
|
|
|
|
if (m_is_in_error_state)
|
|
return;
|
|
m_is_in_error_state = true;
|
|
if (on_error)
|
|
on_error(move(error));
|
|
}
|
|
|
|
void PlaybackManager::set_time_provider(NonnullRefPtr<MediaTimeProvider> const& provider)
|
|
{
|
|
auto time = current_time();
|
|
provider->set_time(time);
|
|
m_time_provider = provider;
|
|
for (auto& track_data : m_video_track_datas) {
|
|
if (!track_data.display)
|
|
continue;
|
|
track_data.display->set_time_provider(provider);
|
|
}
|
|
if (is_playing())
|
|
provider->resume();
|
|
}
|
|
|
|
void PlaybackManager::disable_audio()
|
|
{
|
|
m_audio_sink = nullptr;
|
|
set_time_provider(make_ref_counted<GenericTimeProvider>());
|
|
|
|
for (auto const& track : m_audio_tracks)
|
|
track_stopped_buffering(track);
|
|
}
|
|
|
|
NonnullRefPtr<DisplayingVideoSink> PlaybackManager::get_or_create_the_displaying_video_sink_for_track(Track const& track)
|
|
{
|
|
auto& track_data = get_video_data_for_track(track);
|
|
if (track_data.display == nullptr) {
|
|
track_data.display = MUST(Media::DisplayingVideoSink::try_create(m_time_provider));
|
|
track_data.display->set_provider(track, track_data.provider);
|
|
track_data.display->m_on_start_buffering = [this, track] {
|
|
track_started_buffering(track);
|
|
};
|
|
m_tracks_still_buffering.set(track);
|
|
m_handler->on_track_enabled(track);
|
|
}
|
|
|
|
VERIFY(track_data.display->provider(track) == track_data.provider);
|
|
return *track_data.display;
|
|
}
|
|
|
|
void PlaybackManager::remove_the_displaying_video_sink_for_track(Track const& track)
|
|
{
|
|
auto& track_data = get_video_data_for_track(track);
|
|
VERIFY(track_data.display);
|
|
track_data.display->set_provider(track, nullptr);
|
|
track_data.display = nullptr;
|
|
track_stopped_buffering(track);
|
|
m_handler->on_track_disabled(track);
|
|
}
|
|
|
|
void PlaybackManager::enable_an_audio_track(Track const& track)
|
|
{
|
|
auto& track_data = get_audio_data_for_track(track);
|
|
VERIFY(!track_data.enabled);
|
|
track_data.enabled = true;
|
|
if (m_audio_sink) {
|
|
VERIFY(m_audio_sink->provider(track) == nullptr);
|
|
m_audio_sink->set_provider(track, track_data.provider);
|
|
m_tracks_still_buffering.set(track);
|
|
}
|
|
m_handler->on_track_enabled(track);
|
|
}
|
|
|
|
void PlaybackManager::disable_an_audio_track(Track const& track)
|
|
{
|
|
auto& track_data = get_audio_data_for_track(track);
|
|
VERIFY(track_data.enabled);
|
|
if (m_audio_sink) {
|
|
VERIFY(m_audio_sink->provider(track) == track_data.provider);
|
|
m_audio_sink->set_provider(track, nullptr);
|
|
}
|
|
track_data.enabled = false;
|
|
track_stopped_buffering(track);
|
|
m_handler->on_track_disabled(track);
|
|
}
|
|
|
|
bool PlaybackManager::track_is_enabled(Track const& track) const
|
|
{
|
|
if (track.type() == TrackType::Video) {
|
|
auto const& track_data = get_video_data_for_track(track);
|
|
return track_data.display != nullptr;
|
|
}
|
|
|
|
VERIFY(track.type() == TrackType::Audio);
|
|
auto const& track_data = get_audio_data_for_track(track);
|
|
if (!track_data.enabled)
|
|
return false;
|
|
if (m_audio_sink)
|
|
VERIFY(track_data.provider == m_audio_sink->provider(track));
|
|
return true;
|
|
}
|
|
|
|
void PlaybackManager::start()
|
|
{
|
|
m_handler->start();
|
|
}
|
|
|
|
void PlaybackManager::play()
|
|
{
|
|
m_handler->play();
|
|
}
|
|
|
|
void PlaybackManager::pause()
|
|
{
|
|
m_handler->pause();
|
|
}
|
|
|
|
void PlaybackManager::seek(AK::Duration timestamp, SeekMode mode)
|
|
{
|
|
m_handler->seek(timestamp, mode);
|
|
m_is_in_error_state = false;
|
|
}
|
|
|
|
bool PlaybackManager::is_playing()
|
|
{
|
|
return m_handler->is_playing();
|
|
}
|
|
|
|
PlaybackState PlaybackManager::state()
|
|
{
|
|
return m_handler->state();
|
|
}
|
|
|
|
AvailableData PlaybackManager::available_data()
|
|
{
|
|
return m_handler->available_data();
|
|
}
|
|
|
|
TimeRanges PlaybackManager::buffered_time_ranges() const
|
|
{
|
|
TimeRanges intersection;
|
|
|
|
auto intersect_ranges = [&](auto const& track_datas) {
|
|
for (auto const& track_data : track_datas) {
|
|
if (!track_is_enabled(track_data.track))
|
|
continue;
|
|
|
|
auto range = track_data.provider->buffered_time_ranges();
|
|
if (intersection.is_empty()) {
|
|
intersection = range;
|
|
continue;
|
|
}
|
|
intersection = intersection.intersection(range);
|
|
}
|
|
};
|
|
intersect_ranges(m_video_track_datas);
|
|
intersect_ranges(m_audio_track_datas);
|
|
|
|
return intersection;
|
|
}
|
|
|
|
void PlaybackManager::set_volume(double volume)
|
|
{
|
|
if (m_audio_sink)
|
|
m_audio_sink->set_volume(volume);
|
|
}
|
|
|
|
}
|