/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ use std::cell::Cell; use std::sync::Arc; use dom_struct::dom_struct; use euclid::default::Size2D; use html5ever::{LocalName, Prefix, local_name, ns}; use js::rust::HandleObject; use layout_api::{HTMLMediaData, MediaMetadata}; use net_traits::image_cache::{ ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse, PendingImageId, }; use net_traits::request::{CredentialsMode, Destination, RequestBuilder, RequestId}; use net_traits::{ CoreResourceThread, FetchMetadata, FetchResponseMsg, NetworkError, ResourceFetchTiming, }; use pixels::{Snapshot, SnapshotAlphaMode, SnapshotPixelFormat}; use servo_media::player::video::VideoFrame; use servo_url::ServoUrl; use style::attr::{AttrValue, LengthOrPercentageOrAuto}; use crate::document_loader::{LoadBlocker, LoadType}; use crate::dom::attr::Attr; use crate::dom::bindings::cell::DomRefCell; use crate::dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods; use crate::dom::bindings::inheritance::Castable; use crate::dom::bindings::refcounted::Trusted; use crate::dom::bindings::reflector::DomGlobal; use crate::dom::bindings::root::{DomRoot, LayoutDom}; use crate::dom::bindings::str::DOMString; use crate::dom::csp::{GlobalCspReporting, Violation}; use crate::dom::document::Document; use crate::dom::element::{AttributeMutation, Element, LayoutElementHelpers}; use crate::dom::globalscope::GlobalScope; use crate::dom::html::htmlmediaelement::{HTMLMediaElement, NetworkState, ReadyState}; use crate::dom::node::{Node, NodeDamage, NodeTraits}; use crate::dom::performance::performanceresourcetiming::InitiatorType; use crate::dom::virtualmethods::VirtualMethods; use crate::fetch::{FetchCanceller, RequestWithGlobalScope}; use crate::network_listener::{self, FetchResponseListener, ResourceTimingListener}; use crate::script_runtime::CanGc; #[dom_struct] pub(crate) struct HTMLVideoElement { htmlmediaelement: HTMLMediaElement, /// video_width: Cell>, /// video_height: Cell>, /// Incremented whenever tasks associated with this element are cancelled. generation_id: Cell, /// Load event blocker. Will block the load event while the poster frame /// is being fetched. load_blocker: DomRefCell>, /// A copy of the last frame #[ignore_malloc_size_of = "VideoFrame"] #[no_trace] last_frame: DomRefCell>, } impl HTMLVideoElement { fn new_inherited( local_name: LocalName, prefix: Option, document: &Document, ) -> HTMLVideoElement { HTMLVideoElement { htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document), video_width: Cell::new(None), video_height: Cell::new(None), generation_id: Cell::new(0), load_blocker: Default::default(), last_frame: Default::default(), } } pub(crate) fn new( local_name: LocalName, prefix: Option, document: &Document, proto: Option, can_gc: CanGc, ) -> DomRoot { Node::reflect_node_with_proto( Box::new(HTMLVideoElement::new_inherited( local_name, prefix, document, )), document, proto, can_gc, ) } pub(crate) fn get_video_width(&self) -> Option { self.video_width.get() } pub(crate) fn get_video_height(&self) -> Option { self.video_height.get() } pub(crate) fn set_natural_dimensions(&self, width: Option, height: Option) -> bool { if self.video_width.get() == width && self.video_height.get() == height { return false; } self.video_width.set(width); self.video_height.set(height); self.upcast::().dirty(NodeDamage::Other); true } /// Gets the copy of the video frame at the current playback position, /// if that is available, or else (e.g. when the video is seeking or buffering) /// its previous appearance, if any. pub(crate) fn get_current_frame_data(&self) -> Option { let frame = self.htmlmediaelement.get_current_frame(); if frame.is_some() { *self.last_frame.borrow_mut() = frame; } match self.last_frame.borrow().as_ref() { Some(frame) => { let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32); if !frame.is_gl_texture() { let alpha_mode = SnapshotAlphaMode::Transparent { premultiplied: false, }; Some(Snapshot::from_vec( size.cast(), SnapshotPixelFormat::BGRA, alpha_mode, frame.get_data().to_vec(), )) } else { // XXX(victor): here we only have the GL texture ID. Some(Snapshot::cleared(size.cast())) } }, None => None, } } /// fn update_poster_frame(&self, poster_url: Option<&str>, can_gc: CanGc) { // Step 1. If there is an existing instance of this algorithm running // for this video element, abort that instance of this algorithm without // changing the poster frame. self.generation_id.set(self.generation_id.get() + 1); // Step 2. If the poster attribute's value is the empty string or // if the attribute is absent, then there is no poster frame; return. let Some(poster_url) = poster_url.filter(|poster_url| !poster_url.is_empty()) else { self.htmlmediaelement.set_poster_frame(None); return; }; // Step 3. Let url be the result of encoding-parsing a URL given // the poster attribute's value, relative to the element's node // document. // Step 4. If url is failure, then return. There is no poster frame. let poster_url = match self.owner_document().encoding_parse_a_url(poster_url) { Ok(url) => url, Err(_) => { self.htmlmediaelement.set_poster_frame(None); return; }, }; // We use the image cache for poster frames so we save as much // network activity as possible. let window = self.owner_window(); let image_cache = window.image_cache(); let cache_result = image_cache.get_cached_image_status( poster_url.clone(), window.origin().immutable().clone(), None, ); let id = match cache_result { ImageCacheResult::Available(ImageOrMetadataAvailable::ImageAvailable { image, url, .. }) => { self.process_image_response(ImageResponse::Loaded(image, url), can_gc); return; }, ImageCacheResult::Available(ImageOrMetadataAvailable::MetadataAvailable(_, id)) => id, ImageCacheResult::ReadyForRequest(id) => { self.do_fetch_poster_frame(poster_url, id, can_gc); id }, ImageCacheResult::FailedToLoadOrDecode => { self.process_image_response(ImageResponse::FailedToLoadOrDecode, can_gc); return; }, ImageCacheResult::Pending(id) => id, }; let trusted_node = Trusted::new(self); let generation = self.generation_id(); let callback = window.register_image_cache_listener(id, move |response| { let element = trusted_node.root(); // Ignore any image response for a previous request that has been discarded. if generation != element.generation_id() { return; } element.process_image_response(response.response, CanGc::note()); }); image_cache.add_listener(ImageLoadListener::new(callback, window.pipeline_id(), id)); } /// fn do_fetch_poster_frame(&self, poster_url: ServoUrl, id: PendingImageId, can_gc: CanGc) { // Step 5. Let request be a new request whose URL is url, client is the element's node // document's relevant settings object, destination is "image", initiator type is "video", // credentials mode is "include", and whose use-URL-credentials flag is set. let document = self.owner_document(); let global = self.owner_global(); let request = RequestBuilder::new( Some(document.webview_id()), poster_url.clone(), global.get_referrer(), ) .destination(Destination::Image) .credentials_mode(CredentialsMode::Include) .use_url_credentials(true) .with_global_scope(&global); // Step 6. Fetch request. This must delay the load event of the element's node document. // This delay must be independent from the ones created by HTMLMediaElement during // its media load algorithm, otherwise a code like // // (which triggers no media load algorithm unless a explicit call to .load() is done) // will block the document's load event forever. let blocker = &self.load_blocker; LoadBlocker::terminate(blocker, can_gc); *blocker.borrow_mut() = Some(LoadBlocker::new( &self.owner_document(), LoadType::Image(poster_url.clone()), )); let context = PosterFrameFetchContext::new( self, poster_url, id, request.id, self.global().core_resource_thread(), ); self.owner_document().fetch_background(request, context); } fn generation_id(&self) -> u32 { self.generation_id.get() } /// fn process_image_response(&self, response: ImageResponse, can_gc: CanGc) { // Step 7. If an image is thus obtained, the poster frame is that image. // Otherwise, there is no poster frame. match response { ImageResponse::Loaded(image, url) => { debug!("Loaded poster image for video element: {:?}", url); match image.as_raster_image() { Some(image) => self.htmlmediaelement.set_poster_frame(Some(image)), None => warn!("Vector images are not yet supported in video poster"), } LoadBlocker::terminate(&self.load_blocker, can_gc); }, ImageResponse::MetadataLoaded(..) => {}, // The image cache may have loaded a placeholder for an invalid poster url ImageResponse::FailedToLoadOrDecode => { self.htmlmediaelement.set_poster_frame(None); // A failed load should unblock the document load. LoadBlocker::terminate(&self.load_blocker, can_gc); }, } } /// pub(crate) fn is_usable(&self) -> bool { !matches!( self.htmlmediaelement.get_ready_state(), ReadyState::HaveNothing | ReadyState::HaveMetadata ) } pub(crate) fn origin_is_clean(&self) -> bool { self.htmlmediaelement.origin_is_clean() } pub(crate) fn is_network_state_empty(&self) -> bool { self.htmlmediaelement.network_state() == NetworkState::Empty } } impl HTMLVideoElementMethods for HTMLVideoElement { // make_dimension_uint_getter!(Width, "width"); // make_dimension_uint_setter!(SetWidth, "width"); // make_dimension_uint_getter!(Height, "height"); // make_dimension_uint_setter!(SetHeight, "height"); /// fn VideoWidth(&self) -> u32 { if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing { return 0; } self.video_width.get().unwrap_or(0) } /// fn VideoHeight(&self) -> u32 { if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing { return 0; } self.video_height.get().unwrap_or(0) } // https://html.spec.whatwg.org/multipage/#dom-video-poster make_getter!(Poster, "poster"); // https://html.spec.whatwg.org/multipage/#dom-video-poster make_setter!(SetPoster, "poster"); // For testing purposes only. This is not an event from // https://html.spec.whatwg.org/multipage/#dom-video-poster event_handler!(postershown, GetOnpostershown, SetOnpostershown); } impl VirtualMethods for HTMLVideoElement { fn super_type(&self) -> Option<&dyn VirtualMethods> { Some(self.upcast::() as &dyn VirtualMethods) } fn attribute_mutated(&self, attr: &Attr, mutation: AttributeMutation, can_gc: CanGc) { self.super_type() .unwrap() .attribute_mutated(attr, mutation, can_gc); if attr.local_name() == &local_name!("poster") { if let Some(new_value) = mutation.new_value(attr) { self.update_poster_frame(Some(&new_value), CanGc::note()) } else { self.update_poster_frame(None, CanGc::note()) } }; } fn attribute_affects_presentational_hints(&self, attr: &Attr) -> bool { match attr.local_name() { &local_name!("width") | &local_name!("height") => true, _ => self .super_type() .unwrap() .attribute_affects_presentational_hints(attr), } } fn parse_plain_attribute(&self, name: &LocalName, value: DOMString) -> AttrValue { match name { &local_name!("width") | &local_name!("height") => { AttrValue::from_dimension(value.into()) }, _ => self .super_type() .unwrap() .parse_plain_attribute(name, value), } } } struct PosterFrameFetchContext { /// Reference to the script thread image cache. image_cache: Arc, /// The element that initiated the request. elem: Trusted, /// The cache ID for this request. id: PendingImageId, /// True if this response is invalid and should be ignored. cancelled: bool, /// Url for the resource url: ServoUrl, /// A [`FetchCanceller`] for this request. fetch_canceller: FetchCanceller, } impl FetchResponseListener for PosterFrameFetchContext { fn process_request_body(&mut self, _: RequestId) {} fn process_request_eof(&mut self, _: RequestId) { self.fetch_canceller.ignore() } fn process_response( &mut self, request_id: RequestId, metadata: Result, ) { self.image_cache.notify_pending_response( self.id, FetchResponseMsg::ProcessResponse(request_id, metadata.clone()), ); let metadata = metadata.ok().map(|meta| match meta { FetchMetadata::Unfiltered(m) => m, FetchMetadata::Filtered { unsafe_, .. } => unsafe_, }); let status_is_ok = metadata .as_ref() .map_or(true, |m| m.status.in_range(200..300)); if !status_is_ok { self.cancelled = true; self.fetch_canceller.abort(); } } fn process_response_chunk(&mut self, request_id: RequestId, payload: Vec) { if self.cancelled { // An error was received previously, skip processing the payload. return; } self.image_cache.notify_pending_response( self.id, FetchResponseMsg::ProcessResponseChunk(request_id, payload.into()), ); } fn process_response_eof( self, request_id: RequestId, response: Result<(), NetworkError>, timing: ResourceFetchTiming, ) { self.image_cache.notify_pending_response( self.id, FetchResponseMsg::ProcessResponseEOF(request_id, response.clone(), timing.clone()), ); network_listener::submit_timing(&self, &response, &timing, CanGc::note()); } fn process_csp_violations(&mut self, _request_id: RequestId, violations: Vec) { let global = &self.resource_timing_global(); global.report_csp_violations(violations, None, None); } } impl ResourceTimingListener for PosterFrameFetchContext { fn resource_timing_information(&self) -> (InitiatorType, ServoUrl) { let initiator_type = InitiatorType::LocalName( self.elem .root() .upcast::() .local_name() .to_string(), ); (initiator_type, self.url.clone()) } fn resource_timing_global(&self) -> DomRoot { self.elem.root().owner_document().global() } } impl PosterFrameFetchContext { fn new( elem: &HTMLVideoElement, url: ServoUrl, id: PendingImageId, request_id: RequestId, core_resource_thread: CoreResourceThread, ) -> PosterFrameFetchContext { let window = elem.owner_window(); PosterFrameFetchContext { image_cache: window.image_cache(), elem: Trusted::new(elem), id, cancelled: false, url, fetch_canceller: FetchCanceller::new(request_id, false, core_resource_thread), } } } pub(crate) trait LayoutHTMLVideoElementHelpers { fn data(self) -> HTMLMediaData; fn get_width(self) -> LengthOrPercentageOrAuto; fn get_height(self) -> LengthOrPercentageOrAuto; } impl LayoutHTMLVideoElementHelpers for LayoutDom<'_, HTMLVideoElement> { fn data(self) -> HTMLMediaData { let video = self.unsafe_get(); // Get the current frame being rendered. let current_frame = video.htmlmediaelement.get_current_frame_to_present(); // This value represents the natural width and height of the video. // It may exist even if there is no current frame (for example, after the // metadata of the video is loaded). let metadata = video .get_video_width() .zip(video.get_video_height()) .map(|(width, height)| MediaMetadata { width, height }); HTMLMediaData { current_frame, metadata, } } fn get_width(self) -> LengthOrPercentageOrAuto { self.upcast::() .get_attr_for_layout(&ns!(), &local_name!("width")) .map(AttrValue::as_dimension) .cloned() .unwrap_or(LengthOrPercentageOrAuto::Auto) } fn get_height(self) -> LengthOrPercentageOrAuto { self.upcast::() .get_attr_for_layout(&ns!(), &local_name!("height")) .map(AttrValue::as_dimension) .cloned() .unwrap_or(LengthOrPercentageOrAuto::Auto) } }