Send accessibility tree updates to accesskit

Signed-off-by: Luke Warlow <lwarlow@igalia.com>
Signed-off-by: Delan Azabani <dazabani@igalia.com>

Cargo.lock

@@ -18,11 +18,28 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618"
 
+[[package]]
+name = "accessibility_traits"
+version = "0.0.1"
+dependencies = [
+ "accesskit",
+ "base",
+ "crossbeam-channel",
+ "ipc-channel",
+ "malloc_size_of_derive",
+ "rustc-hash 2.1.1",
+ "serde",
+ "servo_malloc_size_of",
+ "strum",
+]
+
 [[package]]
 name = "accesskit"
 version = "0.21.1"
 source = "git+https://github.com/AccessKit/accesskit.git?rev=783c1e8fea164f337a0cd62739fb6e59f159d77c#783c1e8fea164f337a0cd62739fb6e59f159d77c"
 dependencies = [
+ "enumn",
+ "serde",
  "uuid",
 ]
 
@@ -2449,6 +2466,7 @@ dependencies = [
 name = "embedder_traits"
 version = "0.0.1"
 dependencies = [
+ "accessibility_traits",
  "base",
  "bitflags 2.10.0",
  "cookie 0.18.1",
@@ -2531,6 +2549,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "enumn"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "env_filter"
 version = "0.1.4"
@@ -4959,6 +4988,7 @@ dependencies = [
 name = "layout"
 version = "0.0.1"
 dependencies = [
+ "accessibility_traits",
  "accesskit",
  "app_units",
  "atomic_refcell",
@@ -5120,6 +5150,7 @@ dependencies = [
 name = "libservo"
 version = "0.0.1"
 dependencies = [
+ "accessibility_traits",
  "arboard",
  "background_hang_monitor",
  "base",
@@ -7863,6 +7894,7 @@ dependencies = [
 name = "script_traits"
 version = "0.0.1"
 dependencies = [
+ "accessibility_traits",
  "background_hang_monitor_api",
  "base",
  "bluetooth_traits",
@@ -8386,6 +8418,7 @@ dependencies = [
 name = "servoshell"
 version = "0.0.4"
 dependencies = [
+ "accessibility_traits",
  "android_logger",
  "backtrace",
  "base",

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
+    "components/shared/accessibility",
     "components/xpath",
     "ports/servoshell",
     "tests/unit/*",
@@ -23,6 +24,7 @@ publish = false
 rust-version = "1.86.0"
 
 [workspace.dependencies]
+accessibility_traits = { path = "components/shared/accessibility" }
 accesskit = "0.21.1"
 accountable-refcell = "0.2.2"
 aes = "0.8.4"

components/layout/Cargo.toml

@@ -17,6 +17,7 @@ doctest = false
 tracing = ["dep:tracing"]
 
 [dependencies]
+accessibility_traits = { workspace = true }
 accesskit = { workspace = true }
 app_units = { workspace = true }
 atomic_refcell = { workspace = true }

components/layout/accessibility_tree.rs

@@ -5,6 +5,7 @@
 use std::rc::Rc;
 use std::sync::LazyLock;
 
+pub(crate) use accessibility_traits::AccessibilityTree;
 use accesskit::{Node as AxNode, NodeId as AxNodeId, Role, Tree as AxTree};
 use html5ever::{LocalName, local_name};
 use layout_api::wrapper_traits::{LayoutNode, ThreadSafeLayoutNode};
@@ -16,17 +17,13 @@ use style::dom::{NodeInfo, TDocument, TElement, TNode};
 use crate::FragmentTree;
 
 // #[derive(MallocSizeOf)]
-#[derive(Debug)]
-pub(crate) struct AccessibilityTree {
-    ax_nodes: FxHashMap<AxNodeId, AxNode>,
-    ax_tree: AxTree,
-}
+pub(crate) struct AccessibilityTreeCalculator {}
 
-impl AccessibilityTree {
+impl AccessibilityTreeCalculator {
     pub(crate) fn construct(
         document: ServoLayoutDocument<'_>,
         fragment_tree: Rc<FragmentTree>,
-    ) -> Self {
+    ) -> AccessibilityTree {
         let mut ax_nodes: FxHashMap<AxNodeId, AxNode> = FxHashMap::default();
         let ax_document_id = AxNodeId(document.as_node().opaque().0 as u64);
         let ax_document = AxNode::new(Role::Document);
@@ -75,7 +72,7 @@ impl AccessibilityTree {
             }
             ax_nodes.insert(ax_next_id, ax_next);
         }
-        Self {
+        AccessibilityTree {
             ax_nodes,
             ax_tree: AxTree {
                 root: ax_document_id,
@@ -84,9 +81,6 @@ impl AccessibilityTree {
             },
         }
     }
-    pub(crate) fn descendants(&self) -> AxDescendants<'_> {
-        AxDescendants(self, vec![self.ax_tree.root])
-    }
 }
 
 /// Like [`style::dom::DomChildren`], but reversed.
@@ -107,21 +101,6 @@ impl<N: TNode> Iterator for RevDomChildren<N> {
     }
 }
 
-pub(crate) struct AxDescendants<'tree>(&'tree AccessibilityTree, Vec<AxNodeId>);
-impl<'tree> Iterator for AxDescendants<'tree> {
-    type Item = (AxNodeId, &'tree AxNode);
-    fn next(&mut self) -> Option<Self::Item> {
-        let Some(result_id) = self.1.pop() else {
-            return None;
-        };
-        let result_node = self.0.ax_nodes.get(&result_id).unwrap();
-        for child_id in result_node.children().iter().rev() {
-            self.1.push(*child_id);
-        }
-        Some((result_id, result_node))
-    }
-}
-
 /// <https://www.w3.org/TR/html-aam-1.0/#html-element-role-mappings>
 ///
 /// FIXME: converted mechanically for now, so this will have many errors

components/layout/layout_impl.rs

@@ -30,7 +30,7 @@ use layout_api::{
     ReflowPhasesRun, ReflowRequest, ReflowRequestRestyle, ReflowResult, RegisterPropertyError,
     ScrollContainerQueryFlags, ScrollContainerResponse, TrustedNodeAddress,
 };
-use log::{debug, error, trace, warn};
+use log::{debug, error, warn};
 use malloc_size_of::{MallocConditionalSizeOf, MallocSizeOf, MallocSizeOfOps};
 use net_traits::image_cache::ImageCache;
 use parking_lot::{Mutex, RwLock};
@@ -86,7 +86,7 @@ use url::Url;
 use webrender_api::ExternalScrollId;
 use webrender_api::units::{DevicePixel, LayoutVector2D};
 
-use crate::accessibility_tree::AccessibilityTree;
+use crate::accessibility_tree::{AccessibilityTree, AccessibilityTreeCalculator};
 use crate::context::{CachedImageOrError, ImageResolver, LayoutContext};
 use crate::display_list::{
     DisplayListBuilder, HitTest, LargestContentfulPaintCandidateCollector, StackingContextTree,
@@ -1153,11 +1153,15 @@ impl LayoutThread {
             run_layout()
         });
 
-        let accessibility_tree = AccessibilityTree::construct(document, fragment_tree.clone());
-        for (id, node) in accessibility_tree.descendants() {
-            trace!(target: "layout::accessibility_tree", "AX node: {id:?} => {node:?}");
-        }
-        *self.accessibility_tree.borrow_mut() = Some(accessibility_tree);
+        let accessibility_tree =
+            AccessibilityTreeCalculator::construct(document, fragment_tree.clone());
+        self.script_chan
+            .send(ScriptThreadMessage::HackySendAccessibilityTree(
+                self.webview_id,
+                accessibility_tree,
+            ))
+            .expect("TODO: panic message");
+
         *self.fragment_tree.borrow_mut() = Some(fragment_tree);
 
         if self.debug.style_tree {

components/script/messaging.rs

@@ -104,6 +104,7 @@ impl MixedMessage {
                 ScriptThreadMessage::EmbedderControlResponse(id, _) => Some(id.pipeline_id),
                 ScriptThreadMessage::SetUserContents(..) => None,
                 ScriptThreadMessage::DestroyUserContentManager(..) => None,
+                ScriptThreadMessage::HackySendAccessibilityTree(..) => None,
             },
             MixedMessage::FromScript(inner_msg) => match inner_msg {
                 MainThreadScriptMsg::Common(CommonScriptMsg::Task(_, _, pipeline_id, _)) => {

components/script/script_thread.rs

@@ -1923,6 +1923,15 @@ impl ScriptThread {
                     .borrow_mut()
                     .remove(&user_content_manager_id);
             },
+            ScriptThreadMessage::HackySendAccessibilityTree(webview_id, accessibility_tree) => {
+                self.senders
+                    .pipeline_to_embedder_sender
+                    .send(EmbedderMsg::HackyAccessibilityTreeUpdate(
+                        webview_id,
+                        accessibility_tree,
+                    ))
+                    .expect("TODO: panic message");
+            },
         }
     }
 

components/servo/Cargo.toml

@@ -73,6 +73,7 @@ webxr = [
 ]
 
 [dependencies]
+accessibility_traits = { workspace = true }
 background_hang_monitor = { path = "../background_hang_monitor" }
 base = { workspace = true }
 bitflags = { workspace = true }

components/servo/servo.rs

@@ -638,6 +638,13 @@ impl ServoInner {
                     warn!("Failed to respond to GetScreenMetrics: {error}");
                 }
             },
+            EmbedderMsg::HackyAccessibilityTreeUpdate(webview_id, accessibility_tree) => {
+                if let Some(webview) = self.get_webview_handle(webview_id) {
+                    webview
+                        .delegate()
+                        .hacky_accessibility_tree_update(webview, accessibility_tree);
+                }
+            },
         }
     }
 }

components/servo/webview_delegate.rs

@@ -5,6 +5,7 @@
 use std::path::PathBuf;
 use std::rc::Rc;
 
+use accessibility_traits::AccessibilityTree;
 use base::generic_channel::GenericSender;
 use base::id::PipelineId;
 use compositing_traits::rendering_context::RenderingContext;
@@ -974,6 +975,13 @@ pub trait WebViewDelegate {
     /// A console message was logged by content in this [`WebView`].
     /// <https://developer.mozilla.org/en-US/docs/Web/API/Console_API>
     fn show_console_message(&self, _webview: WebView, _level: ConsoleLogLevel, _message: String) {}
+
+    fn hacky_accessibility_tree_update(
+        &self,
+        _webview: WebView,
+        _accessibility_tree: AccessibilityTree,
+    ) {
+    }
 }
 
 pub(crate) struct DefaultWebViewDelegate;

components/shared/accessibility/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "accessibility_traits"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+[lib]
+name = "accessibility_traits"
+path = "lib.rs"
+
+[dependencies]
+accesskit = { workspace = true, features = ["serde"] }
+rustc-hash = { workspace = true }
+base = { workspace = true }
+crossbeam-channel = { workspace = true }
+ipc-channel = { workspace = true }
+malloc_size_of = { workspace = true }
+malloc_size_of_derive = { workspace = true }
+serde = { workspace = true }
+strum = { workspace = true }

components/shared/accessibility/lib.rs

@@ -0,0 +1,34 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+use accesskit::{Node as AxNode, NodeId as AxNodeId, Tree as AxTree};
+use rustc_hash::FxHashMap;
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct AccessibilityTree {
+    pub ax_nodes: FxHashMap<AxNodeId, AxNode>,
+    pub ax_tree: AxTree,
+}
+
+impl AccessibilityTree {
+    pub fn descendants(&self) -> AxDescendants<'_> {
+        AxDescendants(self, vec![self.ax_tree.root])
+    }
+}
+
+pub struct AxDescendants<'tree>(&'tree AccessibilityTree, Vec<AxNodeId>);
+impl<'tree> Iterator for AxDescendants<'tree> {
+    type Item = (AxNodeId, &'tree AxNode);
+    fn next(&mut self) -> Option<Self::Item> {
+        let Some(result_id) = self.1.pop() else {
+            return None;
+        };
+        let result_node = self.0.ax_nodes.get(&result_id).unwrap();
+        for child_id in result_node.children().iter().rev() {
+            self.1.push(*child_id);
+        }
+        Some((result_id, result_node))
+    }
+}

components/shared/embedder/Cargo.toml

@@ -18,6 +18,7 @@ baked-default-resources = []
 gamepad = []
 
 [dependencies]
+accessibility_traits = { workspace = true }
 base = { workspace = true }
 bitflags = { workspace = true }
 cookie = { workspace = true }

components/shared/embedder/lib.rs

@@ -22,6 +22,7 @@ use std::ops::Range;
 use std::path::PathBuf;
 use std::sync::Arc;
 
+use accessibility_traits::AccessibilityTree;
 use base::generic_channel::{GenericCallback, GenericSender, GenericSharedMemory, SendResult};
 use base::id::{PipelineId, WebViewId};
 use crossbeam_channel::Sender;
@@ -530,6 +531,8 @@ pub enum EmbedderMsg {
     /// Inform the embedding layer that a particular `InputEvent` was handled by Servo
     /// and the embedder can continue processing it, if necessary.
     InputEventHandled(WebViewId, InputEventId, InputEventResult),
+    /// Placeholder
+    HackyAccessibilityTreeUpdate(WebViewId, AccessibilityTree),
 }
 
 impl Debug for EmbedderMsg {

components/shared/script/Cargo.toml

@@ -16,6 +16,7 @@ bluetooth = ["bluetooth_traits"]
 webgpu = ["webgpu_traits"]
 
 [dependencies]
+accessibility_traits = { workspace = true }
 background_hang_monitor_api = { workspace = true }
 base = { workspace = true }
 bluetooth_traits = { workspace = true, optional = true }

components/shared/script/lib.rs

@@ -11,6 +11,7 @@
 
 use std::fmt;
 
+use accessibility_traits::AccessibilityTree;
 use base::cross_process_instant::CrossProcessInstant;
 use base::generic_channel::{GenericCallback, GenericReceiver, GenericSender};
 use base::id::{
@@ -305,6 +306,8 @@ pub enum ScriptThreadMessage {
     /// Release all data for the given `UserContentManagerId` from the `ScriptThread`'s
     /// `user_contents_for_manager_id` map.
     DestroyUserContentManager(UserContentManagerId),
+    /// Placeholder
+    HackySendAccessibilityTree(WebViewId, AccessibilityTree),
 }
 
 impl fmt::Debug for ScriptThreadMessage {

ports/servoshell/Cargo.toml

@@ -56,6 +56,7 @@ webgpu = ["libservo/webgpu"]
 webxr = ["libservo/webxr"]
 
 [dependencies]
+accessibility_traits = { workspace = true }
 bpaf = { version = "0.9.20", features = ["derive"] }
 cfg-if = { workspace = true }
 crossbeam-channel = { workspace = true }

ports/servoshell/desktop/gui.rs

@@ -7,11 +7,12 @@ use std::rc::Rc;
 use std::sync::Arc;
 
 use dpi::PhysicalSize;
+use egui::accesskit::{Role, TreeId, TreeUpdate, Uuid};
 use egui::text::{CCursor, CCursorRange};
 use egui::text_edit::TextEditState;
 use egui::{
     Button, Key, Label, LayerId, Modifiers, PaintCallback, TopBottomPanel, Vec2, WidgetInfo,
-    WidgetType, pos2,
+    WidgetType, accesskit, pos2,
 };
 use egui_glow::{CallbackFn, EguiGlow};
 use egui_winit::EventResponse;
@@ -464,6 +465,18 @@ impl Gui {
             // If the top parts of the GUI changed size, then update the size of the WebView and also
             // the size of its RenderingContext.
             let rect = ctx.available_rect();
+            let tree_id = TreeId(Uuid::from_bytes([1; 16]));
+            let id = egui::Id::new("webview");
+            ctx.accesskit_node_builder(id, |node| {
+                node.set_role(Role::Group);
+                node.set_tree_id(tree_id);
+                node.set_bounds(accesskit::Rect {
+                    x0: rect.left() as f64,
+                    y0: rect.top() as f64,
+                    x1: rect.right() as f64,
+                    y1: rect.bottom() as f64,
+                });
+            });
             let size = Size2D::new(rect.width(), rect.height()) * scale;
             if let Some(webview) = window.active_webview() &&
                 size != webview.size()
@@ -608,6 +621,12 @@ impl Gui {
     pub(crate) fn set_zoom_factor(&self, factor: f32) {
         self.context.egui_ctx.set_zoom_factor(factor);
     }
+
+    pub(crate) fn hacky_accessibility_tree_update(&mut self, updater: impl FnOnce() -> TreeUpdate) {
+        if let Some(adapter) = self.context.egui_winit.accesskit.as_mut() {
+            adapter.update_if_active(updater);
+        }
+    }
 }
 
 fn embedder_image_to_egui_image(image: &Image) -> egui::ColorImage {

ports/servoshell/desktop/headed_window.rs

@@ -13,6 +13,8 @@ use std::env;
 use std::rc::Rc;
 use std::time::Duration;
 
+use accessibility_traits::AccessibilityTree;
+use egui::accesskit::{Node, NodeId, TreeId, TreeUpdate, Uuid};
 use euclid::{Angle, Length, Point2D, Rect, Rotation3D, Scale, Size2D, UnknownUnit, Vector3D};
 use keyboard_types::ShortcutMatcher;
 use log::{debug, info};
@@ -1131,6 +1133,26 @@ impl PlatformWindow for HeadedWindow {
         println!("{message}");
         log::log!(level.into(), "{message}");
     }
+
+    fn hacky_accessibility_tree_update(
+        &self,
+        _webview: WebView,
+        accessibility_tree: AccessibilityTree,
+    ) {
+        let nodes: Vec<(NodeId, Node)> = accessibility_tree
+            .ax_nodes
+            .iter()
+            .map(|(k, v)| (*k, v.clone()))
+            .collect();
+        self.gui
+            .borrow_mut()
+            .hacky_accessibility_tree_update(|| TreeUpdate {
+                nodes,
+                tree: Some(accessibility_tree.ax_tree),
+                tree_id: TreeId(Uuid::from_bytes([1; 16])),
+                focus: NodeId(1),
+            });
+    }
 }
 
 fn winit_phase_to_touch_event_type(phase: TouchPhase) -> TouchEventType {

ports/servoshell/egl/app.rs

@@ -4,11 +4,12 @@
 use std::cell::{Cell, RefCell};
 use std::rc::Rc;
 
+use accessibility_traits::AccessibilityTree;
 use base::generic_channel::GenericCallback;
 use dpi::PhysicalSize;
 use euclid::{Rect, Scale};
 use keyboard_types::{CompositionEvent, CompositionState, Key, KeyState, NamedKey};
-use log::{info, warn};
+use log::{info, trace, warn};
 use raw_window_handle::{RawWindowHandle, WindowHandle};
 use servo::{
     DeviceIndependentIntRect, DeviceIndependentPixel, DeviceIntSize, DevicePixel, DevicePoint,

ports/servoshell/running_app_state.rs

@@ -9,6 +9,7 @@ use std::collections::HashMap;
 use std::collections::hash_map::Entry;
 use std::rc::Rc;
 
+use accessibility_traits::AccessibilityTree;
 use crossbeam_channel::{Receiver, Sender, unbounded};
 use euclid::Rect;
 use image::{DynamicImage, ImageFormat, RgbaImage};
@@ -819,6 +820,15 @@ impl WebViewDelegate for RunningAppState {
         self.platform_window_for_webview_id(webview.id())
             .show_console_message(level, &message);
     }
+
+    fn hacky_accessibility_tree_update(
+        &self,
+        webview: WebView,
+        accessibility_tree: AccessibilityTree,
+    ) {
+        self.platform_window_for_webview_id(webview.id())
+            .hacky_accessibility_tree_update(webview, accessibility_tree);
+    }
 }
 
 struct ServoShellServoDelegate;

ports/servoshell/window.rs

@@ -5,6 +5,7 @@
 use std::cell::{Cell, RefCell};
 use std::rc::Rc;
 
+use accessibility_traits::AccessibilityTree;
 use euclid::Scale;
 use log::warn;
 use servo::{
@@ -433,4 +434,6 @@ pub(crate) trait PlatformWindow {
     fn as_headed_window(&self) -> Option<&crate::egl::app::EmbeddedPlatformWindow> {
         None
     }
+
+    fn hacky_accessibility_tree_update(&self, _: WebView, _: AccessibilityTree) {}
 }