servoshell: Hook up Android software keyboard to embedder events. (#40009)

This is the most basic integration possible. Current limitations
include:
* the done button doesn't trigger form submission/keyboard hiding
* IME events don't trigger inputs (ie. pressing and holding a letter to
get more options)

However, it is infinitely better than the current integration.

Testing: Manually tested in the Android emulator.
Fixes: #12127 (we can open more specific issues after this)

Signed-off-by: Josh Matthews <josh@joshmatthews.net>

ports/servoshell/egl/android.rs

@@ -15,6 +15,7 @@ use android_logger::{self, Config, FilterBuilder};
 use jni::objects::{GlobalRef, JClass, JObject, JString, JValue, JValueOwned};
 use jni::sys::{jboolean, jfloat, jint, jobject};
 use jni::{JNIEnv, JavaVM};
+use keyboard_types::{Key, NamedKey};
 use log::{debug, error, info, warn};
 use raw_window_handle::{
     AndroidDisplayHandle, AndroidNdkWindowHandle, RawDisplayHandle, RawWindowHandle,
@@ -48,7 +49,7 @@ pub extern "C" fn android_main() {
 
 fn call<F>(env: &mut JNIEnv, f: F)
 where
-    F: Fn(&RunningAppState),
+    F: FnOnce(&RunningAppState),
 {
     APP.with(|app| match app.borrow().as_ref() {
         Some(ref app_state) => (f)(app_state),
@@ -257,6 +258,81 @@ pub extern "C" fn Java_org_servo_servoview_JNIServo_scroll<'local>(
     call(&mut env, |s| s.scroll(dx as f32, dy as f32, x, y));
 }
 
+enum KeyCode {
+    Delete,
+    ForwardDelete,
+    Enter,
+    ArrowLeft,
+    ArrowRight,
+    ArrowUp,
+    ArrowDown,
+}
+
+impl TryFrom<i32> for KeyCode {
+    type Error = ();
+
+    // Values derived from <https://developer.android.com/reference/android/view/KeyEvent>
+    fn try_from(keycode: i32) -> Result<KeyCode, ()> {
+        Ok(match keycode {
+            66 => KeyCode::Enter,
+            67 => KeyCode::Delete,
+            112 => KeyCode::ForwardDelete,
+            21 => KeyCode::ArrowLeft,
+            22 => KeyCode::ArrowRight,
+            19 => KeyCode::ArrowUp,
+            20 => KeyCode::ArrowDown,
+            _ => return Err(()),
+        })
+    }
+}
+
+impl From<KeyCode> for Key {
+    fn from(keycode: KeyCode) -> Key {
+        Key::Named(match keycode {
+            KeyCode::Enter => NamedKey::Enter,
+            KeyCode::Delete => NamedKey::Backspace,
+            KeyCode::ForwardDelete => NamedKey::Delete,
+            KeyCode::ArrowLeft => NamedKey::ArrowLeft,
+            KeyCode::ArrowRight => NamedKey::ArrowRight,
+            KeyCode::ArrowUp => NamedKey::ArrowUp,
+            KeyCode::ArrowDown => NamedKey::ArrowDown,
+        })
+    }
+}
+
+fn key_from_unicode_keycode(unicode: u32, keycode: i32) -> Option<Key> {
+    char::from_u32(unicode)
+        .filter(|c| *c != '\0')
+        .map(|c| Key::Character(String::from(c)))
+        .or_else(|| KeyCode::try_from(keycode).ok().map(Key::from))
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Java_org_servo_servoview_JNIServo_keydown<'local>(
+    mut env: JNIEnv<'local>,
+    _: JClass<'local>,
+    keycode: jint,
+    unicode: jint,
+) {
+    debug!("keydown {keycode}");
+    if let Some(key) = key_from_unicode_keycode(unicode as u32, keycode) {
+        call(&mut env, move |s| s.key_down(key));
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Java_org_servo_servoview_JNIServo_keyup<'local>(
+    mut env: JNIEnv<'local>,
+    _: JClass<'local>,
+    keycode: jint,
+    unicode: jint,
+) {
+    debug!("keyup {keycode}");
+    if let Some(key) = key_from_unicode_keycode(unicode as u32, keycode) {
+        call(&mut env, move |s| s.key_up(key));
+    }
+}
+
 #[unsafe(no_mangle)]
 pub extern "C" fn Java_org_servo_servoview_JNIServo_touchDown<'local>(
     mut env: JNIEnv<'local>,
@@ -595,8 +671,16 @@ impl HostTrait for HostCallbacks {
         _multiline: bool,
         _rect: DeviceIntRect,
     ) {
+        let mut env = self.jvm.get_env().unwrap();
+        env.call_method(self.callbacks.as_obj(), "onImeShow", "()V", &[])
+            .unwrap();
+    }
+
+    fn on_ime_hide(&self) {
+        let mut env = self.jvm.get_env().unwrap();
+        env.call_method(self.callbacks.as_obj(), "onImeHide", "()V", &[])
+            .unwrap();
     }
-    fn on_ime_hide(&self) {}
 
     fn on_media_session_metadata(&self, title: String, artist: String, album: String) {
         info!("on_media_session_metadata");

support/android/apk/servoapp/src/main/java/org/servo/servoshell/MainActivity.java

@@ -14,6 +14,7 @@ import android.os.Bundle;
 import android.system.ErrnoException;
 import android.system.Os;
 import android.util.Log;
+import android.view.KeyEvent;
 import android.view.View;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputMethodManager;
@@ -38,6 +39,7 @@ public class MainActivity extends Activity implements Servo.Client {
     ImageButton mReloadButton;
     ImageButton mStopButton;
     EditText mUrlField;
+    boolean mUrlFieldIsFocused;
     ProgressBar mProgressBar;
     TextView mIdleText;
     boolean mCanGoBack;
@@ -54,6 +56,7 @@ public class MainActivity extends Activity implements Servo.Client {
         mReloadButton = findViewById(R.id.reloadbutton);
         mStopButton = findViewById(R.id.stopbutton);
         mUrlField = findViewById(R.id.urlfield);
+        mUrlFieldIsFocused = false;
         mProgressBar = findViewById(R.id.progressbar);
         mIdleText = findViewById(R.id.redrawing);
         mCanGoBack = false;
@@ -105,11 +108,12 @@ public class MainActivity extends Activity implements Servo.Client {
             return false;
         });
         mUrlField.setOnFocusChangeListener((v, hasFocus) -> {
-            if (v.getId() == R.id.urlfield && !hasFocus) {
-                InputMethodManager imm =
-                        (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE);
-                assert imm != null;
-                imm.hideSoftInputFromWindow(v.getWindowToken(), 0);
+            if (v.getId() == R.id.urlfield) {
+                mUrlFieldIsFocused = hasFocus;
+                if (!hasFocus) {
+                    InputMethodManager imm = getSystemService(InputMethodManager.class);
+                    imm.hideSoftInputFromWindow(v.getWindowToken(), 0);
+                }
             }
         });
     }
@@ -138,6 +142,34 @@ public class MainActivity extends Activity implements Servo.Client {
         mServoView.stop();
     }
 
+    @Override
+    public void onImeShow() {
+        InputMethodManager imm = getSystemService(InputMethodManager.class);
+        imm.showSoftInput(mServoView, InputMethodManager.SHOW_IMPLICIT);
+    }
+
+    @Override
+    public void onImeHide() {
+        InputMethodManager imm = getSystemService(InputMethodManager.class);
+        imm.hideSoftInputFromWindow(mServoView.getWindowToken(), InputMethodManager.SHOW_IMPLICIT);
+    }
+
+    @Override
+    public boolean onKeyDown(int keyCode, KeyEvent event) {
+        if (mUrlFieldIsFocused) {
+            return true;
+        }
+        return mServoView.onKeyDown(keyCode, event);
+    }
+
+    @Override
+    public boolean onKeyUp(int keyCode, KeyEvent event) {
+        if (mUrlFieldIsFocused) {
+            return true;
+        }
+        return mServoView.onKeyUp(keyCode, event);
+    }
+
     @Override
     public void onAlert(String message) {
         AlertDialog.Builder builder = new AlertDialog.Builder(this);

support/android/apk/servoview/src/main/java/org/servo/servoview/JNIServo.java

@@ -43,6 +43,9 @@ public class JNIServo {
 
     public native void scroll(int dx, int dy, int x, int y);
 
+    public native void keydown(int keycode, int unicode);
+    public native void keyup(int keycode, int unicode);
+
     public native void touchDown(float x, float y, int pointer_id);
 
     public native void touchMove(float x, float y, int pointer_id);
@@ -106,6 +109,9 @@ public class JNIServo {
 
         void onShutdownComplete();
 
+        void onImeShow();
+        void onImeHide();
+
         void onMediaSessionMetadata(String title, String artist, String album);
 
         void onMediaSessionPlaybackStateChange(int state);

support/android/apk/servoview/src/main/java/org/servo/servoview/Servo.java

@@ -6,6 +6,7 @@
 package org.servo.servoview;
 
 import android.app.Activity;
+import android.view.KeyEvent;
 import android.view.Surface;
 
 import org.servo.servoview.JNIServo.ServoCoordinates;
@@ -111,6 +112,14 @@ public class Servo {
         mRunCallback.inGLThread(() -> mJNI.scroll(dx, dy, x, y));
     }
 
+    public void onKeyDown(int keyCode, KeyEvent event) {
+        mRunCallback.inGLThread(() -> mJNI.keydown(keyCode, event.getUnicodeChar()));
+    }
+
+    public void onKeyUp(int keyCode, KeyEvent event) {
+        mRunCallback.inGLThread(() -> mJNI.keyup(keyCode, event.getUnicodeChar()));
+    }
+
     public void touchDown(float x, float y, int pointerId) {
         mRunCallback.inGLThread(() -> mJNI.touchDown(x, y, pointerId));
     }
@@ -175,6 +184,9 @@ public class Servo {
 
         void onRedrawing(boolean redrawing);
 
+        void onImeShow();
+        void onImeHide();
+
         void onMediaSessionMetadata(String title, String artist, String album);
 
         void onMediaSessionPlaybackStateChange(int state);
@@ -234,6 +246,14 @@ public class Servo {
             mShutdownComplete = true;
         }
 
+        public void onImeShow() {
+            mRunCallback.inUIThread(() -> mClient.onImeShow());
+        }
+
+        public void onImeHide() {
+            mRunCallback.inUIThread(() -> mClient.onImeHide());
+        }
+
         public void onAnimatingChanged(boolean animating) {
             mRunCallback.inGLThread(() -> mGfxCb.animationStateChanged(animating));
         }

support/android/apk/servoview/src/main/java/org/servo/servoview/ServoView.java

@@ -13,6 +13,7 @@ import android.os.Handler;
 import android.os.Looper;
 import android.util.DisplayMetrics;
 import android.util.Log;
+import android.view.KeyEvent;
 import android.view.Surface;
 import android.view.SurfaceView;
 import android.view.SurfaceHolder;
@@ -229,6 +230,16 @@ public class ServoView extends SurfaceView
         }
     }
 
+    public boolean onKeyDown(int keyCode, KeyEvent event) {
+        mServo.onKeyDown(keyCode, event);
+        return true;
+    }
+
+    public boolean onKeyUp(int keyCode, KeyEvent event) {
+        mServo.onKeyUp(keyCode, event);
+        return true;
+    }
+
     public void scroll(int dx, int dy, int x, int y) {
         mServo.scroll(dx, dy, x, y);
     }