From af331e3bff04c0e821e9fe223d24ea4d8a6f2914 Mon Sep 17 00:00:00 2001 From: Robert Bragg Date: Sun, 25 Jun 2023 21:29:38 +0100 Subject: [PATCH] Rework input_events API and expose KeyCharacterMap bindings With the way events are delivered via an `InputQueue` with `NativeActivity` there is no direct access to the underlying KeyEvent and MotionEvent Java objects and no `ndk` API that supports the equivalent of `KeyEvent.getUnicodeChar()` What `getUnicodeChar` does under the hood though is to do lookups into a `KeyCharacterMap` for the corresponding `InputDevice` based on the event's `key_code` and `meta_state` - which are things we can do via some JNI bindings for `KeyCharacterMap`. Although it's still awkward to expose an API like `key_event.get_unicode_char()` we can instead provide an API that lets you look up a `KeyCharacterMap` for any `device_id` and applications can then use that for character mapping. This approach is also more general than the `getUnicodeChar` utility since it exposes other useful state, such as being able to check what kind of keyboard input events are coming from (such as a full physical keyboard vs a virtual / 'predictive' keyboard) For consistency this exposes the same API through the game-activity backend, even though the game-activity backend is technically able to support unicode lookups via `getUnicodeChar` (since it has access to the Java `KeyEvent` object). This highlighted a need to be able to use other `AndroidApp` APIs while processing input, which wasn't possible with the `.input_events()` API design because the `AndroidApp` held a lock over the backend while iterating events. This changes `input_events()` to `input_events_iter()` which now returns a form of lending iterator and instead of taking a callback that gets called repeatedly by `input_events()` a similar callback is now passed to `iter.next(callback)`. The API isn't as ergonomic as I would have liked, considering that lending iterators aren't a standard feature for Rust yet but also since we still want to have the handling for each individual event go via a callback that can report whether an event was "handled". I think the slightly awkward ergonomics are acceptable though considering that the API will generally be used as an implementation detail within middleware frameworks like Winit. Since this is the first example where we're creating non-trivial Java bindings for an Android SDK API this adds some JNI utilities and establishes a pattern for how we can implement a class binding. It's an implementation detail but with how I wrote the binding I tried to keep in mind the possibility of creating a procmacro later that would generate some of the JNI boilerplate involved. --- android-activity/CHANGELOG.md | 98 +++- android-activity/Cargo.toml | 2 + android-activity/LICENSE | 24 +- android-activity/src/error.rs | 58 +++ android-activity/src/game_activity/input.rs | 8 +- android-activity/src/game_activity/mod.rs | 445 ++++++++++++------ android-activity/src/input.rs | 23 + android-activity/src/input/sdk.rs | 358 ++++++++++++++ android-activity/src/jni_utils.rs | 151 ++++++ android-activity/src/lib.rs | 165 ++++++- android-activity/src/native_activity/glue.rs | 22 +- android-activity/src/native_activity/input.rs | 9 + android-activity/src/native_activity/mod.rs | 189 ++++++-- examples/agdk-mainloop/src/lib.rs | 129 ++++- examples/na-mainloop/src/lib.rs | 129 ++++- 15 files changed, 1567 insertions(+), 243 deletions(-) create mode 100644 android-activity/src/error.rs create mode 100644 android-activity/src/input/sdk.rs create mode 100644 android-activity/src/jni_utils.rs diff --git a/android-activity/CHANGELOG.md b/android-activity/CHANGELOG.md index 35afefa..dd66796 100644 --- a/android-activity/CHANGELOG.md +++ b/android-activity/CHANGELOG.md @@ -1,12 +1,108 @@ - + # Changelog The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added +- Added `KeyEvent::meta_state()` for being able to query the state of meta keys, needed for character mapping ([#102](https://github.com/rust-mobile/android-activity/pull/102)) +- Added `KeyCharacterMap` JNI bindings to the corresponding Android SDK API ([#102](https://github.com/rust-mobile/android-activity/pull/102)) +- Added `AndroidApp::device_key_character_map()` for being able to get a `KeyCharacterMap` for a given `device_id` for unicode character mapping ([#102](https://github.com/rust-mobile/android-activity/pull/102)) + +
+ Click here for an example of how to handle unicode character mapping: + + ```rust + let mut combining_accent = None; + // Snip + + + let combined_key_char = if let Ok(map) = app.device_key_character_map(device_id) { + match map.get(key_event.key_code(), key_event.meta_state()) { + Ok(KeyMapChar::Unicode(unicode)) => { + let combined_unicode = if let Some(accent) = combining_accent { + match map.get_dead_char(accent, unicode) { + Ok(Some(key)) => { + info!("KeyEvent: Combined '{unicode}' with accent '{accent}' to give '{key}'"); + Some(key) + } + Ok(None) => None, + Err(err) => { + log::error!("KeyEvent: Failed to combine 'dead key' accent '{accent}' with '{unicode}': {err:?}"); + None + } + } + } else { + info!("KeyEvent: Pressed '{unicode}'"); + Some(unicode) + }; + combining_accent = None; + combined_unicode.map(|unicode| KeyMapChar::Unicode(unicode)) + } + Ok(KeyMapChar::CombiningAccent(accent)) => { + info!("KeyEvent: Pressed 'dead key' combining accent '{accent}'"); + combining_accent = Some(accent); + Some(KeyMapChar::CombiningAccent(accent)) + } + Ok(KeyMapChar::None) => { + info!("KeyEvent: Pressed non-unicode key"); + combining_accent = None; + None + } + Err(err) => { + log::error!("KeyEvent: Failed to get key map character: {err:?}"); + combining_accent = None; + None + } + } + } else { + None + }; + ``` + +
+ ### Changed - GameActivity updated to 2.0.2 (requires the corresponding 2.0.2 `.aar` release from Google) ([#88](https://github.com/rust-mobile/android-activity/pull/88)) +- `AndroidApp::input_events()` is replaced by `AndroidApp::input_events_iter()` ([#102](https://github.com/rust-mobile/android-activity/pull/102)) + +
+ Click here for an example of how to use `input_events_iter()`: + + ```rust + match app.input_events_iter() { + Ok(mut iter) => { + loop { + let read_input = iter.next(|event| { + let handled = match event { + InputEvent::KeyEvent(key_event) => { + // Snip + } + InputEvent::MotionEvent(motion_event) => { + // Snip + } + event => { + // Snip + } + }; + + handled + }); + + if !read_input { + break; + } + } + } + Err(err) => { + log::error!("Failed to get input events iterator: {err:?}"); + } + } + ``` + +
## [0.4.3] - 2022-07-30 ### Fixed diff --git a/android-activity/Cargo.toml b/android-activity/Cargo.toml index 5a01bd6..9233f4c 100644 --- a/android-activity/Cargo.toml +++ b/android-activity/Cargo.toml @@ -38,6 +38,7 @@ native-activity = [] log = "0.4" jni-sys = "0.3" cesu8 = "1" +jni = "0.21" ndk = "0.7" ndk-sys = "0.4" ndk-context = "0.1" @@ -45,6 +46,7 @@ android-properties = "0.2" num_enum = "0.6" bitflags = "2.0" libc = "0.2" +thiserror = "1" [build-dependencies] cc = { version = "1.0", features = ["parallel"] } diff --git a/android-activity/LICENSE b/android-activity/LICENSE index b4635e9..47bdfd8 100644 --- a/android-activity/LICENSE +++ b/android-activity/LICENSE @@ -1,12 +1,24 @@ -The third-party glue code, under the native-activity-csrc/ and game-activity-csrc/ directories -is covered by the Apache 2.0 license only: +# License -Apache License, Version 2.0 (docs/LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) +## GameActivity +The third-party glue code, under the game-activity-csrc/ directory is covered by +the Apache 2.0 license only: + +Apache License, Version 2.0 (docs/LICENSE-APACHE or ) + +## SDK Documentation + +Documentation for APIs that are direct bindings of Android platform APIs are covered +by the Apache 2.0 license only: + +Apache License, Version 2.0 (docs/LICENSE-APACHE or ) + +## android-activity All other code is dual-licensed under either -* MIT License (docs/LICENSE-MIT or http://opensource.org/licenses/MIT) -* Apache License, Version 2.0 (docs/LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) +- MIT License (docs/LICENSE-MIT or ) +- Apache License, Version 2.0 (docs/LICENSE-APACHE or ) -at your option. \ No newline at end of file +at your option. diff --git a/android-activity/src/error.rs b/android-activity/src/error.rs new file mode 100644 index 0000000..7193f5a --- /dev/null +++ b/android-activity/src/error.rs @@ -0,0 +1,58 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AppError { + #[error("Operation only supported from the android_main() thread: {0}")] + NonMainThread(String), + + #[error("Java VM or JNI error, including Java exceptions")] + JavaError(String), + + #[error("Input unavailable")] + InputUnavailable, +} + +pub type Result = std::result::Result; + +// XXX: we don't want to expose jni-rs in the public API +// so we have an internal error type that we can generally +// use in the backends and then we can strip the error +// in the frontend of the API. +// +// This way we avoid exposing a public trait implementation for +// `From` +#[derive(Error, Debug)] +pub(crate) enum InternalAppError { + #[error("A JNI error")] + JniError(jni::errors::JniError), + #[error("A Java Exception was thrown via a JNI method call")] + JniException(String), + #[error("A Java VM error")] + JvmError(jni::errors::Error), + #[error("Input unavailable")] + InputUnavailable, +} + +pub(crate) type InternalResult = std::result::Result; + +impl From for InternalAppError { + fn from(value: jni::errors::Error) -> Self { + InternalAppError::JvmError(value) + } +} +impl From for InternalAppError { + fn from(value: jni::errors::JniError) -> Self { + InternalAppError::JniError(value) + } +} + +impl From for AppError { + fn from(value: InternalAppError) -> Self { + match value { + InternalAppError::JniError(err) => AppError::JavaError(err.to_string()), + InternalAppError::JniException(msg) => AppError::JavaError(msg), + InternalAppError::JvmError(err) => AppError::JavaError(err.to_string()), + InternalAppError::InputUnavailable => AppError::InputUnavailable, + } + } +} diff --git a/android-activity/src/game_activity/input.rs b/android-activity/src/game_activity/input.rs index ba442b7..29b33bf 100644 --- a/android-activity/src/game_activity/input.rs +++ b/android-activity/src/game_activity/input.rs @@ -16,7 +16,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; use std::convert::TryInto; -use crate::game_activity::ffi::{GameActivityKeyEvent, GameActivityMotionEvent}; +use crate::activity_impl::ffi::{GameActivityKeyEvent, GameActivityMotionEvent}; use crate::input::{Class, Source}; // Note: try to keep this wrapper API compatible with the AInputEvent API if possible @@ -1274,6 +1274,12 @@ impl<'a> KeyEvent<'a> { action.try_into().unwrap() } + #[inline] + pub fn action_button(&self) -> KeyAction { + let action = self.ga_event.action as u32; + action.try_into().unwrap() + } + /// Returns the last time the key was pressed. This is on the scale of /// `java.lang.System.nanoTime()`, which has nanosecond precision, but no defined start time. /// diff --git a/android-activity/src/game_activity/mod.rs b/android-activity/src/game_activity/mod.rs index fd9da25..88a9fd6 100644 --- a/android-activity/src/game_activity/mod.rs +++ b/android-activity/src/game_activity/mod.rs @@ -1,5 +1,6 @@ #![cfg(feature = "game-activity")] +use std::collections::HashMap; use std::ffi::{CStr, CString}; use std::fs::File; use std::io::{BufRead, BufReader}; @@ -8,7 +9,8 @@ use std::ops::Deref; use std::os::unix::prelude::*; use std::panic::catch_unwind; use std::ptr::NonNull; -use std::sync::{Arc, RwLock}; +use std::sync::Weak; +use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; use std::{ptr, thread}; @@ -24,6 +26,9 @@ use ndk::asset::AssetManager; use ndk::configuration::Configuration; use ndk::native_window::NativeWindow; +use crate::error::InternalResult; +use crate::input::{KeyCharacterMap, KeyCharacterMapBinding}; +use crate::jni_utils::{self, CloneJavaVM}; use crate::util::{abort_on_panic, android_log, log_panic}; use crate::{ util, AndroidApp, ConfigurationRef, InputStatus, MainEvent, PollEvent, Rect, WindowManagerFlags, @@ -90,7 +95,7 @@ impl<'a> StateLoader<'a> { if !(*app_ptr).savedState.is_null() && (*app_ptr).savedStateSize > 0 { let buf: &mut [u8] = std::slice::from_raw_parts_mut( (*app_ptr).savedState.cast(), - (*app_ptr).savedStateSize as usize, + (*app_ptr).savedStateSize, ); let state = buf.to_vec(); Some(state) @@ -120,7 +125,16 @@ impl AndroidAppWaker { } impl AndroidApp { - pub(crate) unsafe fn from_ptr(ptr: NonNull) -> Self { + pub(crate) unsafe fn from_ptr(ptr: NonNull, jvm: CloneJavaVM) -> Self { + let mut env = jvm.get_env().unwrap(); // We attach to the thread before creating the AndroidApp + + let key_map_binding = match KeyCharacterMapBinding::new(&mut env) { + Ok(b) => b, + Err(err) => { + panic!("Failed to create KeyCharacterMap JNI bindings: {err:?}"); + } + }; + // Note: we don't use from_ptr since we don't own the android_app.config // and need to keep in mind that the Drop handler is going to call // AConfiguration_delete() @@ -128,15 +142,19 @@ impl AndroidApp { Self { inner: Arc::new(RwLock::new(AndroidAppInner { + jvm, native_app: NativeAppGlue { ptr }, config: ConfigurationRef::new(config), native_window: Default::default(), + key_map_binding: Arc::new(key_map_binding), + key_maps: Mutex::new(HashMap::new()), + input_receiver: Mutex::new(None), })), } } } -#[derive(Debug)] +#[derive(Debug, Clone)] struct NativeAppGlue { ptr: NonNull, } @@ -150,11 +168,112 @@ impl Deref for NativeAppGlue { unsafe impl Send for NativeAppGlue {} unsafe impl Sync for NativeAppGlue {} +impl NativeAppGlue { + // TODO: move into a trait + pub fn text_input_state(&self) -> TextInputState { + unsafe { + let activity = (*self.as_ptr()).activity; + let mut out_state = TextInputState { + text: String::new(), + selection: TextSpan { start: 0, end: 0 }, + compose_region: None, + }; + let out_ptr = &mut out_state as *mut TextInputState; + + let app_ptr = self.as_ptr(); + (*app_ptr).textInputState = 0; + + // NEON WARNING: + // + // It's not clearly documented but the GameActivity API over the + // GameTextInput library directly exposes _modified_ UTF8 text + // from Java so we need to be careful to convert text to and + // from UTF8 + // + // GameTextInput also uses a pre-allocated, fixed-sized buffer for + // the current text state and has shared `currentState_` that + // appears to have no lock to guard access from multiple threads. + // + // There's also no locking at the GameActivity level, so I'm fairly + // certain that `GameActivity_getTextInputState` isn't thread + // safe: https://issuetracker.google.com/issues/294112477 + // + // Overall this is all quite gnarly - and probably a good reminder + // of why we want to use Rust instead of C/C++. + ffi::GameActivity_getTextInputState( + activity, + Some(AndroidAppInner::map_input_state_to_text_event_callback), + out_ptr.cast(), + ); + + out_state + } + } + + // TODO: move into a trait + pub fn set_text_input_state(&self, state: TextInputState) { + unsafe { + let activity = (*self.as_ptr()).activity; + let modified_utf8 = cesu8::to_java_cesu8(&state.text); + let text_length = modified_utf8.len() as i32; + let modified_utf8_bytes = modified_utf8.as_ptr(); + let ffi_state = ffi::GameTextInputState { + text_UTF8: modified_utf8_bytes.cast(), // NB: may be signed or unsigned depending on target + text_length, + selection: ffi::GameTextInputSpan { + start: state.selection.start as i32, + end: state.selection.end as i32, + }, + composingRegion: match state.compose_region { + Some(span) => { + // The GameText subclass of InputConnection only has a special case for removing the + // compose region if `start == -1` but the docs for `setComposingRegion` imply that + // the region should effectively be removed if any empty region is given (unlike for the + // selection region, it's not meaningful to maintain an empty compose region) + // + // We aim for more consistent behaviour by normalizing any empty region into `(-1, -1)` + // to remove the compose region. + // + // NB `setComposingRegion` itself is documented to clamp start/end to the text bounds + // so apart from this special-case handling in GameText's implementation of + // `setComposingRegion` then there's nothing special about `(-1, -1)` - it's just an empty + // region that should get clamped to `(0, 0)` and then get removed. + if span.start == span.end { + ffi::GameTextInputSpan { start: -1, end: -1 } + } else { + ffi::GameTextInputSpan { + start: span.start as i32, + end: span.end as i32, + } + } + } + None => ffi::GameTextInputSpan { start: -1, end: -1 }, + }, + }; + ffi::GameActivity_setTextInputState(activity, &ffi_state as *const _); + } + } +} + #[derive(Debug)] pub struct AndroidAppInner { + pub(crate) jvm: CloneJavaVM, native_app: NativeAppGlue, config: ConfigurationRef, native_window: RwLock>, + + /// Shared JNI bindings for the `KeyCharacterMap` class + key_map_binding: Arc, + + /// A table of `KeyCharacterMap`s per `InputDevice` ID + /// these are used to be able to map key presses to unicode + /// characters + key_maps: Mutex>, + + /// While an app is reading input events it holds an + /// InputReceiver reference which we track to ensure + /// we don't hand out more than one receiver at a time + input_receiver: Mutex>>, } impl AndroidAppInner { @@ -370,9 +489,9 @@ impl AndroidAppInner { let text_modified_utf8: *const u8 = (*state).text_UTF8.cast(); let text_modified_utf8 = std::slice::from_raw_parts(text_modified_utf8, (*state).text_length as usize); - match cesu8::from_java_cesu8(&text_modified_utf8) { + match cesu8::from_java_cesu8(text_modified_utf8) { Ok(str) => { - let len = *&str.len(); + let len = str.len(); (*out_ptr).text = String::from(str); let selection_start = (*state).selection.start.clamp(0, len as i32 + 1); @@ -398,82 +517,34 @@ impl AndroidAppInner { // TODO: move into a trait pub fn text_input_state(&self) -> TextInputState { - unsafe { - let activity = (*self.native_app.as_ptr()).activity; - let mut out_state = TextInputState { - text: String::new(), - selection: TextSpan { start: 0, end: 0 }, - compose_region: None, - }; - let out_ptr = &mut out_state as *mut TextInputState; - - // NEON WARNING: - // - // It's not clearly documented but the GameActivity API over the - // GameTextInput library directly exposes _modified_ UTF8 text - // from Java so we need to be careful to convert text to and - // from UTF8 - // - // GameTextInput also uses a pre-allocated, fixed-sized buffer for the current - // text state but GameTextInput doesn't actually provide it's own thread - // safe API to safely access this state so we have to cooperate with - // the GameActivity code that does locking when reading/writing the state - // (I.e. we can't just punch through to the GameTextInput layer from here). - // - // Overall this is all quite gnarly - and probably a good reminder of why - // we want to use Rust instead of C/C++. - ffi::GameActivity_getTextInputState( - activity, - Some(AndroidAppInner::map_input_state_to_text_event_callback), - out_ptr.cast(), - ); - - out_state - } + self.native_app.text_input_state() } // TODO: move into a trait pub fn set_text_input_state(&self, state: TextInputState) { - unsafe { - let activity = (*self.native_app.as_ptr()).activity; - let modified_utf8 = cesu8::to_java_cesu8(&state.text); - let text_length = modified_utf8.len() as i32; - let modified_utf8_bytes = modified_utf8.as_ptr(); - let ffi_state = ffi::GameTextInputState { - text_UTF8: modified_utf8_bytes.cast(), // NB: may be signed or unsigned depending on target - text_length, - selection: ffi::GameTextInputSpan { - start: state.selection.start as i32, - end: state.selection.end as i32, - }, - composingRegion: match state.compose_region { - Some(span) => { - // The GameText subclass of InputConnection only has a special case for removing the - // compose region if `start == -1` but the docs for `setComposingRegion` imply that - // the region should effectively be removed if any empty region is given (unlike for the - // selection region, it's not meaningful to maintain an empty compose region) - // - // We aim for more consistent behaviour by normalizing any empty region into `(-1, -1)` - // to remove the compose region. - // - // NB `setComposingRegion` itself is documented to clamp start/end to the text bounds - // so apart from this special-case handling in GameText's implementation of - // `setComposingRegion` then there's nothing special about `(-1, -1)` - it's just an empty - // region that should get clamped to `(0, 0)` and then get removed. - if span.start == span.end { - ffi::GameTextInputSpan { start: -1, end: -1 } - } else { - ffi::GameTextInputSpan { - start: span.start as i32, - end: span.end as i32, - } - } - } - None => ffi::GameTextInputSpan { start: -1, end: -1 }, - }, - }; - ffi::GameActivity_setTextInputState(activity, &ffi_state as *const _); - } + self.native_app.set_text_input_state(state); + } + + pub(crate) fn device_key_character_map( + &self, + device_id: i32, + ) -> InternalResult { + let mut guard = self.key_maps.lock().unwrap(); + + let key_map = match guard.entry(device_id) { + std::collections::hash_map::Entry::Occupied(occupied) => occupied.get().clone(), + std::collections::hash_map::Entry::Vacant(vacant) => { + let character_map = jni_utils::device_key_character_map( + self.jvm.clone(), + self.key_map_binding.clone(), + device_id, + )?; + vacant.insert(character_map.clone()); + character_map + } + }; + + Ok(key_map) } pub fn enable_motion_axis(&mut self, axis: Axis) { @@ -519,49 +590,26 @@ impl AndroidAppInner { } } - fn dispatch_key_and_motion_events(&self, mut callback: F) - where - F: FnMut(&InputEvent) -> InputStatus, - { - let buf = unsafe { - let app_ptr = self.native_app.as_ptr(); - let input_buffer = ffi::android_app_swap_input_buffers(app_ptr); - if input_buffer.is_null() { - return; - } - InputBuffer::from_ptr(NonNull::new_unchecked(input_buffer)) - }; + pub(crate) fn input_events_receiver(&self) -> InternalResult> { + let mut guard = self.input_receiver.lock().unwrap(); - let mut keys_iter = KeyEventsLendingIterator::new(&buf); - while let Some(key_event) = keys_iter.next() { - callback(&InputEvent::KeyEvent(key_event)); - } - let mut motion_iter = MotionEventsLendingIterator::new(&buf); - while let Some(motion_event) = motion_iter.next() { - callback(&InputEvent::MotionEvent(motion_event)); - } - } - - fn dispatch_text_events(&self, mut callback: F) - where - F: FnMut(&InputEvent) -> InputStatus, - { - unsafe { - let app_ptr = self.native_app.as_ptr(); - if (*app_ptr).textInputState != 0 { - let state = self.text_input_state(); - callback(&InputEvent::TextEvent(state)); - (*app_ptr).textInputState = 0; + // Make sure we don't hand out more than one receiver at a time because + // turning the reciever into an interator will perform a swap_buffers + // for the buffered input events which shouldn't happen while we're in + // the middle of iterating events + if let Some(receiver) = &*guard { + if receiver.strong_count() > 0 { + return Err(crate::error::InternalAppError::InputUnavailable); } } - } + *guard = None; - pub fn input_events(&self, mut callback: F) - where - F: FnMut(&InputEvent) -> InputStatus, - { - self.dispatch_key_and_motion_events(&mut callback); - self.dispatch_text_events(&mut callback); + let receiver = Arc::new(InputReceiver { + native_app: self.native_app.clone(), + }); + + *guard = Some(Arc::downgrade(&receiver)); + Ok(receiver) } pub fn internal_data_path(&self) -> Option { @@ -586,33 +634,28 @@ impl AndroidAppInner { } } -struct MotionEventsLendingIterator<'a> { +struct MotionEventsLendingIterator { pos: usize, count: usize, - buffer: &'a InputBuffer<'a>, } -// A kind of lending iterator but since our MSRV is 1.60 we can't handle this -// via a generic trait. The iteration of motion events is entirely private -// though so this is ok for now. -impl<'a> MotionEventsLendingIterator<'a> { - fn new(buffer: &'a InputBuffer<'a>) -> Self { +impl MotionEventsLendingIterator { + fn new(buffer: &InputBuffer) -> Self { Self { pos: 0, count: buffer.motion_events_count(), - buffer, } } - fn next(&mut self) -> Option> { + fn next<'buf>(&mut self, buffer: &'buf InputBuffer) -> Option> { if self.pos < self.count { // Safety: // - This iterator currently has exclusive access to the front buffer of events // - We know the buffer is non-null // - `pos` is less than the number of events stored in the buffer let ga_event = unsafe { - (*self.buffer.ptr.as_ptr()) + (*buffer.ptr.as_ptr()) .motionEvents - .offset(self.pos as isize) + .add(self.pos) .as_ref() .unwrap() }; @@ -625,33 +668,28 @@ impl<'a> MotionEventsLendingIterator<'a> { } } -struct KeyEventsLendingIterator<'a> { +struct KeyEventsLendingIterator { pos: usize, count: usize, - buffer: &'a InputBuffer<'a>, } -// A kind of lending iterator but since our MSRV is 1.60 we can't handle this -// via a generic trait. The iteration of key events is entirely private -// though so this is ok for now. -impl<'a> KeyEventsLendingIterator<'a> { - fn new(buffer: &'a InputBuffer<'a>) -> Self { +impl KeyEventsLendingIterator { + fn new(buffer: &InputBuffer) -> Self { Self { pos: 0, count: buffer.key_events_count(), - buffer, } } - fn next(&mut self) -> Option> { + fn next<'buf>(&mut self, buffer: &'buf InputBuffer) -> Option> { if self.pos < self.count { // Safety: // - This iterator currently has exclusive access to the front buffer of events // - We know the buffer is non-null // - `pos` is less than the number of events stored in the buffer let ga_event = unsafe { - (*self.buffer.ptr.as_ptr()) + (*buffer.ptr.as_ptr()) .keyEvents - .offset(self.pos as isize) + .add(self.pos) .as_ref() .unwrap() }; @@ -673,7 +711,7 @@ impl<'a> InputBuffer<'a> { pub(crate) fn from_ptr(ptr: NonNull) -> InputBuffer<'a> { Self { ptr, - _lifetime: PhantomData::default(), + _lifetime: PhantomData, } } @@ -695,6 +733,118 @@ impl<'a> Drop for InputBuffer<'a> { } } +/// Conceptually we can think of this like the receiver end of an +/// input events channel. +/// +/// After being passed back to AndroidApp it gets turned into a +/// lending iterator for pending input events. +/// +/// It serves two purposes: +/// 1. It represents an exclusive access to input events (the application +/// can only have one receiver at a time) and it's intended to support +/// the double-buffering design for input events in GameActivity where +/// we issue a swap_buffers before iterating events and wouldn't want +/// another swap to be possible before finishing - especially since +/// we want to borrow directly from the buffer while dispatching. +/// 2. It doesn't borrow from AndroidAppInner so we can pass it back to +/// AndroidApp which can drop its lock around AndroidAppInner and +/// it can then be turned into a lending iterator. (We wouldn't +/// be able to pass the iterator back to the application if it +/// borrowed from within the lock and we need to drop the lock +/// because otherwise the app wouldn't be able to access the AndroidApp +/// API in any way while iterating events) +#[derive(Debug)] +pub(crate) struct InputReceiver { + // Safety: the native_app effectively has a static lifetime and it + // has its own internal locking when calling + // `android_app_swap_input_buffers` + native_app: NativeAppGlue, +} + +impl<'a> From> for InputIteratorInner<'a> { + fn from(receiver: Arc) -> Self { + let buffered = unsafe { + let app_ptr = receiver.native_app.as_ptr(); + let input_buffer = ffi::android_app_swap_input_buffers(app_ptr); + if input_buffer.is_null() { + None + } else { + let buffer = InputBuffer::from_ptr(NonNull::new_unchecked(input_buffer)); + let keys_iter = KeyEventsLendingIterator::new(&buffer); + let motion_iter = MotionEventsLendingIterator::new(&buffer); + Some(BufferedEvents::<'a> { + buffer, + keys_iter, + motion_iter, + }) + } + }; + + let native_app = receiver.native_app.clone(); + Self { + _receiver: receiver, + buffered, + native_app, + text_event_checked: false, + } + } +} + +struct BufferedEvents<'a> { + buffer: InputBuffer<'a>, + keys_iter: KeyEventsLendingIterator, + motion_iter: MotionEventsLendingIterator, +} + +pub(crate) struct InputIteratorInner<'a> { + // Held to maintain exclusive access to buffered input events + _receiver: Arc, + + buffered: Option>, + native_app: NativeAppGlue, + text_event_checked: bool, +} + +impl<'a> InputIteratorInner<'a> { + pub(crate) fn next(&mut self, callback: F) -> bool + where + F: FnOnce(&input::InputEvent) -> InputStatus, + { + if let Some(buffered) = &mut self.buffered { + if let Some(key_event) = buffered.keys_iter.next(&buffered.buffer) { + let _ = callback(&InputEvent::KeyEvent(key_event)); + return true; + } + if let Some(motion_event) = buffered.motion_iter.next(&buffered.buffer) { + let _ = callback(&InputEvent::MotionEvent(motion_event)); + return true; + } + self.buffered = None; + } + + if !self.text_event_checked { + self.text_event_checked = true; + unsafe { + let app_ptr = self.native_app.as_ptr(); + + // XXX: It looks like the GameActivity implementation should + // be using atomic ops to set this flag, and require us to + // use atomics to check and clear it too. + // + // We currently just hope that with the lack of atomic ops that + // the compiler isn't reordering code so this gets flagged + // before the java main thread really updates the state. + if (*app_ptr).textInputState != 0 { + let state = self.native_app.text_input_state(); // Will clear .textInputState + let _ = callback(&InputEvent::TextEvent(state)); + return true; + } + } + } + false + } +} + // Rust doesn't give us a clean way to directly export symbols from C/C++ // so we rename the C/C++ symbols and re-export these JNI entrypoints from // Rust... @@ -789,19 +939,16 @@ pub unsafe extern "C" fn _rust_glue_entry(native_app: *mut ffi::android_app) { let activity: jobject = (*(*native_app).activity).javaGameActivity; ndk_context::initialize_android_context(jvm.cast(), activity.cast()); + let jvm = CloneJavaVM::from_raw(jvm).unwrap(); // Since this is a newly spawned thread then the JVM hasn't been attached // to the thread yet. Attach before calling the applications main function // so they can safely make JNI calls - let mut jenv_out: *mut core::ffi::c_void = std::ptr::null_mut(); - if let Some(attach_current_thread) = (*(*jvm)).AttachCurrentThread { - attach_current_thread(jvm, &mut jenv_out, std::ptr::null_mut()); - } - + jvm.attach_current_thread_permanently().unwrap(); jvm }; unsafe { - let app = AndroidApp::from_ptr(NonNull::new(native_app).unwrap()); + let app = AndroidApp::from_ptr(NonNull::new(native_app).unwrap(), jvm.clone()); // We want to specifically catch any panic from the application's android_main // so we can finish + destroy the Activity gracefully via the JVM @@ -824,9 +971,9 @@ pub unsafe extern "C" fn _rust_glue_entry(native_app: *mut ffi::android_app) { // to the main thread of the process where the Java finish call will take place" ffi::GameActivity_finish((*native_app).activity); - if let Some(detach_current_thread) = (*(*jvm)).DetachCurrentThread { - detach_current_thread(jvm); - } + // This should detach automatically but lets detach explicitly to avoid depending + // on the TLS trickery in `jni-rs` + jvm.detach_current_thread(); ndk_context::release_android_context(); } diff --git a/android-activity/src/input.rs b/android-activity/src/input.rs index b9edf7e..f62a088 100644 --- a/android-activity/src/input.rs +++ b/android-activity/src/input.rs @@ -2,6 +2,10 @@ use bitflags::bitflags; use num_enum::{IntoPrimitive, TryFromPrimitive}; pub use crate::activity_impl::input::*; +use crate::InputStatus; + +mod sdk; +pub use sdk::*; /// An enum representing the source of an [`MotionEvent`] or [`KeyEvent`] /// @@ -120,3 +124,22 @@ pub struct TextInputState { /// If the resulting region is zero-sized, no region is marked (equivalent to passing `None`) pub compose_region: Option, } + +/// An exclusive, lending iterator for input events +pub struct InputIterator<'a> { + pub(crate) inner: crate::activity_impl::InputIteratorInner<'a>, +} + +impl<'a> InputIterator<'a> { + /// Reads and handles the next input event by passing it to the given `callback` + /// + /// `callback` should return [`InputStatus::Unhandled`] for any input events that aren't directly + /// handled by the application, or else [`InputStatus::Handled`]. Unhandled events may lead to a + /// fallback interpretation of the event. + pub fn next(&mut self, callback: F) -> bool + where + F: FnOnce(&crate::activity_impl::input::InputEvent) -> InputStatus, + { + self.inner.next(callback) + } +} diff --git a/android-activity/src/input/sdk.rs b/android-activity/src/input/sdk.rs new file mode 100644 index 0000000..4d1e84b --- /dev/null +++ b/android-activity/src/input/sdk.rs @@ -0,0 +1,358 @@ +use std::sync::Arc; + +use jni::{ + objects::{GlobalRef, JClass, JMethodID, JObject, JStaticMethodID, JValue}, + signature::{Primitive, ReturnType}, + JNIEnv, +}; +use jni_sys::jint; + +use crate::{ + activity_impl::input::{Keycode, MetaState}, + jni_utils::CloneJavaVM, +}; + +use crate::{ + error::{AppError, InternalAppError}, + jni_utils, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum KeyboardType { + /// A numeric (12-key) keyboard. + /// + /// A numeric keyboard supports text entry using a multi-tap approach. It may be necessary to tap a key multiple times to generate the desired letter or symbol. + /// + /// This type of keyboard is generally designed for thumb typing. + Numeric, + + /// A keyboard with all the letters, but with more than one letter per key. + /// + /// This type of keyboard is generally designed for thumb typing. + Predictive, + + /// A keyboard with all the letters, and maybe some numbers. + /// + /// An alphabetic keyboard supports text entry directly but may have a condensed layout with a small form factor. In contrast to a full keyboard, some symbols may only be accessible using special on-screen character pickers. In addition, to improve typing speed and accuracy, the framework provides special affordances for alphabetic keyboards such as auto-capitalization and toggled / locked shift and alt keys. + /// + /// This type of keyboard is generally designed for thumb typing. + Alpha, + + /// A full PC-style keyboard. + /// + /// A full keyboard behaves like a PC keyboard. All symbols are accessed directly by pressing keys on the keyboard without on-screen support or affordances such as auto-capitalization. + /// + /// This type of keyboard is generally designed for full two hand typing. + Full, + + /// A keyboard that is only used to control special functions rather than for typing. + /// + /// A special function keyboard consists only of non-printing keys such as HOME and POWER that are not actually used for typing. + SpecialFunction, + + /// An unknown type of keyboard + Unknown(i32), +} + +impl From for KeyboardType { + fn from(value: i32) -> Self { + match value { + 1 => KeyboardType::Numeric, + 2 => KeyboardType::Predictive, + 3 => KeyboardType::Alpha, + 4 => KeyboardType::Full, + 5 => KeyboardType::SpecialFunction, + unknown => KeyboardType::Unknown(unknown), + } + } +} +impl From for i32 { + fn from(value: KeyboardType) -> i32 { + match value { + KeyboardType::Numeric => 1, + KeyboardType::Predictive => 2, + KeyboardType::Alpha => 3, + KeyboardType::Full => 4, + KeyboardType::SpecialFunction => 5, + KeyboardType::Unknown(unknown) => unknown, + } + } +} + +/// Either represents, a unicode character or combining accent from a +/// [`KeyCharacterMap`], or `None` for non-printable keys. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum KeyMapChar { + None, + Unicode(char), + CombiningAccent(char), +} + +// I've also tried to think here about how to we could potentially automatically +// generate a binding struct like `KeyCharacterMapBinding` with a procmacro and +// so have intentionally limited the `Binding` being a very thin, un-opinionated +// wrapper based on basic JNI types. + +/// Lower-level JNI binding for `KeyCharacterMap` class only holds 'static state +/// and can be shared with an `Arc` ref count. +/// +/// The separation here also neatly helps us separate `InternalAppError` from +/// `AppError` for mapping JNI errors without exposing any `jni-rs` types in the +/// public API. +#[derive(Debug)] +pub(crate) struct KeyCharacterMapBinding { + //vm: JavaVM, + klass: GlobalRef, + get_method_id: JMethodID, + get_dead_char_method_id: JStaticMethodID, + get_keyboard_type_method_id: JMethodID, +} + +impl KeyCharacterMapBinding { + pub(crate) fn new(env: &mut JNIEnv) -> Result { + let binding = env.with_local_frame::<_, _, InternalAppError>(10, |env| { + let klass = env.find_class("android/view/KeyCharacterMap")?; // Creates a local ref + Ok(Self { + get_method_id: env.get_method_id(&klass, "get", "(II)I")?, + get_dead_char_method_id: env.get_static_method_id( + &klass, + "getDeadChar", + "(II)I", + )?, + get_keyboard_type_method_id: env.get_method_id(&klass, "getKeyboardType", "()I")?, + klass: env.new_global_ref(&klass)?, + }) + })?; + Ok(binding) + } + + pub fn get<'local>( + &self, + env: &'local mut JNIEnv, + key_map: impl AsRef>, + key_code: jint, + meta_state: jint, + ) -> Result { + let key_map = key_map.as_ref(); + + // Safety: + // - we know our global `key_map` reference is non-null and valid. + // - we know `get_method_id` remains valid + // - we know that the signature of KeyCharacterMap::get is `(int, int) -> int` + // - we know this won't leak any local references as a side effect + // + // We know it's ok to unwrap the `.i()` value since we explicitly + // specify the return type as `Int` + let unicode = unsafe { + env.call_method_unchecked( + key_map, + self.get_method_id, + ReturnType::Primitive(Primitive::Int), + &[ + JValue::Int(key_code).as_jni(), + JValue::Int(meta_state).as_jni(), + ], + ) + } + .map_err(|err| jni_utils::clear_and_map_exception_to_err(env, err))?; + Ok(unicode.i().unwrap()) + } + + pub fn get_dead_char( + &self, + env: &mut JNIEnv, + accent_char: jint, + base_char: jint, + ) -> Result { + // Safety: + // - we know `get_dead_char_method_id` remains valid + // - we know that KeyCharacterMap::getDeadKey is a static method + // - we know that the signature of KeyCharacterMap::getDeadKey is `(int, int) -> int` + // - we know this won't leak any local references as a side effect + // + // We know it's ok to unwrap the `.i()` value since we explicitly + // specify the return type as `Int` + + // Urgh, it's pretty terrible that there's no ergonomic/safe way to get a JClass reference from a GlobalRef + // Safety: we don't do anything that would try to delete the JClass as if it were a real local reference + let klass = unsafe { JClass::from_raw(self.klass.as_obj().as_raw()) }; + let unicode = unsafe { + env.call_static_method_unchecked( + &klass, + self.get_dead_char_method_id, + ReturnType::Primitive(Primitive::Int), + &[ + JValue::Int(accent_char).as_jni(), + JValue::Int(base_char).as_jni(), + ], + ) + } + .map_err(|err| jni_utils::clear_and_map_exception_to_err(env, err))?; + Ok(unicode.i().unwrap()) + } + + pub fn get_keyboard_type<'local>( + &self, + env: &'local mut JNIEnv, + key_map: impl AsRef>, + ) -> Result { + let key_map = key_map.as_ref(); + + // Safety: + // - we know our global `key_map` reference is non-null and valid. + // - we know `get_keyboard_type_method_id` remains valid + // - we know that the signature of KeyCharacterMap::getKeyboardType is `() -> int` + // - we know this won't leak any local references as a side effect + // + // We know it's ok to unwrap the `.i()` value since we explicitly + // specify the return type as `Int` + Ok(unsafe { + env.call_method_unchecked( + key_map, + self.get_keyboard_type_method_id, + ReturnType::Primitive(Primitive::Int), + &[], + ) + } + .map_err(|err| jni_utils::clear_and_map_exception_to_err(env, err))? + .i() + .unwrap()) + } +} + +/// Describes the keys provided by a keyboard device and their associated labels. +#[derive(Clone, Debug)] +pub struct KeyCharacterMap { + jvm: CloneJavaVM, + binding: Arc, + key_map: GlobalRef, +} + +impl KeyCharacterMap { + pub(crate) fn new( + jvm: CloneJavaVM, + binding: Arc, + key_map: GlobalRef, + ) -> Self { + Self { + jvm, + binding, + key_map, + } + } + + /// Gets the Unicode character generated by the specified [`Keycode`] and [`MetaState`] combination. + /// + /// Returns [`KeyMapChar::None`] if the key is not one that is used to type Unicode characters. + /// + /// Returns [`KeyMapChar::CombiningAccent`] if the key is a "dead key" that should be combined with + /// another to actually produce a character -- see [`KeyCharacterMap::get_dead_char`]. + /// + /// # Errors + /// + /// Since this API needs to use JNI internally to call into the Android JVM it may return + /// a [`AppError::JavaError`] in case there is a spurious JNI error or an exception + /// is caught. + pub fn get(&self, key_code: Keycode, meta_state: MetaState) -> Result { + let key_code: u32 = key_code.into(); + let key_code = key_code as jni_sys::jint; + let meta_state: u32 = meta_state.0; + let meta_state = meta_state as jni_sys::jint; + + // Since we expect this API to be called from the `main` thread then we expect to already be + // attached to the JVM + // + // Safety: there's no other JNIEnv in scope so this env can't be used to subvert the mutable + // borrow rules that ensure we can only add local references to the top JNI frame. + let mut env = self.jvm.get_env().map_err(|err| { + let err: InternalAppError = err.into(); + err + })?; + let unicode = self + .binding + .get(&mut env, self.key_map.as_obj(), key_code, meta_state)?; + let unicode = unicode as u32; + + const COMBINING_ACCENT: u32 = 0x80000000; + const COMBINING_ACCENT_MASK: u32 = !COMBINING_ACCENT; + + if unicode == 0 { + Ok(KeyMapChar::None) + } else if unicode & COMBINING_ACCENT == COMBINING_ACCENT { + let accent = unicode & COMBINING_ACCENT_MASK; + // Safety: assumes Android key maps don't contain invalid unicode characters + Ok(KeyMapChar::CombiningAccent(unsafe { + char::from_u32_unchecked(accent) + })) + } else { + // Safety: assumes Android key maps don't contain invalid unicode characters + Ok(KeyMapChar::Unicode(unsafe { + char::from_u32_unchecked(unicode) + })) + } + } + + /// Get the character that is produced by combining the dead key producing accent with the key producing character c. + /// + /// For example, ```get_dead_char('`', 'e')``` returns 'รจ'. `get_dead_char('^', ' ')` returns '^' and `get_dead_char('^', '^')` returns '^'. + /// + /// # Errors + /// + /// Since this API needs to use JNI internally to call into the Android JVM it may return + /// a [`AppError::JavaError`] in case there is a spurious JNI error or an exception + /// is caught. + pub fn get_dead_char( + &self, + accent_char: char, + base_char: char, + ) -> Result, AppError> { + let accent_char = accent_char as jni_sys::jint; + let base_char = base_char as jni_sys::jint; + + // Since we expect this API to be called from the `main` thread then we expect to already be + // attached to the JVM + // + // Safety: there's no other JNIEnv in scope so this env can't be used to subvert the mutable + // borrow rules that ensure we can only add local references to the top JNI frame. + let mut env = self.jvm.get_env().map_err(|err| { + let err: InternalAppError = err.into(); + err + })?; + let unicode = self + .binding + .get_dead_char(&mut env, accent_char, base_char)?; + let unicode = unicode as u32; + + // Safety: assumes Android key maps don't contain invalid unicode characters + Ok(if unicode == 0 { + None + } else { + Some(unsafe { char::from_u32_unchecked(unicode) }) + }) + } + + /// Gets the keyboard type. + /// + /// Different keyboard types have different semantics. See [`KeyboardType`] for details. + /// + /// # Errors + /// + /// Since this API needs to use JNI internally to call into the Android JVM it may return + /// a [`AppError::JavaError`] in case there is a spurious JNI error or an exception + /// is caught. + pub fn get_keyboard_type(&self) -> Result { + // Since we expect this API to be called from the `main` thread then we expect to already be + // attached to the JVM + // + // Safety: there's no other JNIEnv in scope so this env can't be used to subvert the mutable + // borrow rules that ensure we can only add local references to the top JNI frame. + let mut env = self.jvm.get_env().map_err(|err| { + let err: InternalAppError = err.into(); + err + })?; + let keyboard_type = self + .binding + .get_keyboard_type(&mut env, self.key_map.as_obj())?; + Ok(keyboard_type.into()) + } +} diff --git a/android-activity/src/jni_utils.rs b/android-activity/src/jni_utils.rs new file mode 100644 index 0000000..dcf6083 --- /dev/null +++ b/android-activity/src/jni_utils.rs @@ -0,0 +1,151 @@ +//! The JNI calls we make in this crate are often not part of a Java native +//! method implementation and so we can't assume we have a JNI local frame that +//! is going to unwind and free local references, and we also can't just leave +//! exceptions to get thrown when returning to Java. +//! +//! These utilities help us check + clear exceptions and map them into Rust Errors. + +use std::{ops::Deref, sync::Arc}; + +use jni::{ + objects::{JObject, JString}, + JavaVM, +}; + +use crate::{ + error::{InternalAppError, InternalResult}, + input::{KeyCharacterMap, KeyCharacterMapBinding}, +}; + +// TODO: JavaVM should implement Clone +#[derive(Debug)] +pub(crate) struct CloneJavaVM { + pub jvm: JavaVM, +} +impl Clone for CloneJavaVM { + fn clone(&self) -> Self { + Self { + jvm: unsafe { JavaVM::from_raw(self.jvm.get_java_vm_pointer()).unwrap() }, + } + } +} +impl CloneJavaVM { + pub unsafe fn from_raw(jvm: *mut jni_sys::JavaVM) -> InternalResult { + Ok(Self { + jvm: JavaVM::from_raw(jvm)?, + }) + } +} +unsafe impl Send for CloneJavaVM {} +unsafe impl Sync for CloneJavaVM {} + +impl Deref for CloneJavaVM { + type Target = JavaVM; + + fn deref(&self) -> &Self::Target { + &self.jvm + } +} + +/// Use with `.map_err()` to map `jni::errors::Error::JavaException` into a +/// richer error based on the actual contents of the `JThrowable` +/// +/// (The `jni` crate doesn't do that automatically since it's more +/// common to let the exception get thrown when returning to Java) +/// +/// This will also clear the exception +pub(crate) fn clear_and_map_exception_to_err( + env: &mut jni::JNIEnv<'_>, + err: jni::errors::Error, +) -> InternalAppError { + if matches!(err, jni::errors::Error::JavaException) { + let result = env.with_local_frame::<_, _, InternalAppError>(5, |env| { + let e = env.exception_occurred()?; + assert!(!e.is_null()); // should only be called after receiving a JavaException Result + env.exception_clear()?; + + let class = env.get_object_class(&e)?; + //let get_stack_trace_method = env.get_method_id(&class, "getStackTrace", "()[Ljava/lang/StackTraceElement;")?; + let get_message_method = + env.get_method_id(&class, "getMessage", "()Ljava/lang/String;")?; + + let msg = unsafe { + env.call_method_unchecked( + &e, + get_message_method, + jni::signature::ReturnType::Object, + &[], + )? + .l() + .unwrap() + }; + let msg = unsafe { JString::from_raw(JObject::into_raw(msg)) }; + let msg = env.get_string(&msg)?; + let msg: String = msg.into(); + + // TODO: get Java backtrace: + /* + if let JValue::Object(elements) = env.call_method_unchecked(&e, get_stack_trace_method, jni::signature::ReturnType::Array, &[])? { + let elements = env.auto_local(elements); + + } + */ + + Ok(msg) + }); + + match result { + Ok(msg) => InternalAppError::JniException(msg), + Err(err) => InternalAppError::JniException(format!( + "UNKNOWN (Failed to query JThrowable: {err:?})" + )), + } + } else { + err.into() + } +} + +pub(crate) fn device_key_character_map( + jvm: CloneJavaVM, + key_map_binding: Arc, + device_id: i32, +) -> InternalResult { + // Don't really need to 'attach' since this should be called from the app's main thread that + // should already be attached, but the redundancy should be fine + // + // Attach 'permanently' to avoid any chance of detaching the thread from the VM + let mut env = jvm.attach_current_thread_permanently()?; + + // We don't want to accidentally leak any local references while we + // aren't going to be returning from here back to the JVM, to unwind, so + // we make a local frame + let character_map = env.with_local_frame::<_, _, jni::errors::Error>(10, |env| { + let input_device_class = env.find_class("android/view/InputDevice")?; // Creates a local ref + let device = env + .call_static_method( + input_device_class, + "getDevice", + "(I)Landroid/view/InputDevice;", + &[device_id.into()], + )? + .l()?; // Creates a local ref + + let character_map = env + .call_method( + &device, + "getKeyCharacterMap", + "()Landroid/view/KeyCharacterMap;", + &[], + )? + .l()?; + let character_map = env.new_global_ref(character_map)?; + + Ok(character_map) + })?; + + Ok(KeyCharacterMap::new( + jvm.clone(), + key_map_binding, + character_map, + )) +} diff --git a/android-activity/src/lib.rs b/android-activity/src/lib.rs index e00a902..8b9c46e 100644 --- a/android-activity/src/lib.rs +++ b/android-activity/src/lib.rs @@ -62,6 +62,7 @@ use std::sync::Arc; use std::sync::RwLock; use std::time::Duration; +use input::KeyCharacterMap; use libc::c_void; use ndk::asset::AssetManager; use ndk::native_window::NativeWindow; @@ -96,15 +97,12 @@ You may need to add a `[patch]` into your Cargo.toml to ensure a specific versio android-activity is used across all of your application's crates."# ); -#[cfg(any(feature = "native-activity", doc))] -mod native_activity; -#[cfg(any(feature = "native-activity", doc))] -use native_activity as activity_impl; +#[cfg_attr(any(feature = "native-activity", doc), path = "native_activity/mod.rs")] +#[cfg_attr(any(feature = "game-activity", doc), path = "game_activity/mod.rs")] +pub(crate) mod activity_impl; -#[cfg(feature = "game-activity")] -mod game_activity; -#[cfg(feature = "game-activity")] -use game_activity as activity_impl; +pub mod error; +use error::Result; pub mod input; @@ -113,6 +111,8 @@ pub use config::ConfigurationRef; mod util; +mod jni_utils; + /// A rectangle with integer edge coordinates. Used to represent window insets, for example. #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct Rect { @@ -163,14 +163,14 @@ pub use activity_impl::StateSaver; #[non_exhaustive] #[derive(Debug)] pub enum MainEvent<'a> { - /// New input events are available via [`AndroidApp::input_events()`] + /// New input events are available via [`AndroidApp::input_events_iter()`] /// /// _Note: Even if more input is received this event will not be resent - /// until [`AndroidApp::input_events()`] has been called, which enables + /// until [`AndroidApp::input_events_iter()`] has been called, which enables /// applications to batch up input processing without there being lots of /// redundant event loop wake ups._ /// - /// [`AndroidApp::input_events()`]: AndroidApp::input_events + /// [`AndroidApp::input_events_iter()`]: AndroidApp::input_events_iter InputAvailable, /// Command from main thread: a new [`NativeWindow`] is ready for use. Upon @@ -629,24 +629,139 @@ impl AndroidApp { self.inner.read().unwrap().set_text_input_state(state); } - /// Query and process all out-standing input event + /// Get an exclusive, lending iterator over buffered input events /// - /// `callback` should return [`InputStatus::Unhandled`] for any input events that aren't directly - /// handled by the application, or else [`InputStatus::Handled`]. Unhandled events may lead to a - /// fallback interpretation of the event. + /// Applications are expected to call this in-sync with their rendering or + /// in response to a [`MainEvent::InputAvailable`] event being delivered. /// - /// Applications are generally either expected to call this in-sync with their rendering or - /// in response to a [`MainEvent::InputAvailable`] event being delivered. _Note though that your - /// application is will only be delivered a single [`MainEvent::InputAvailable`] event between calls - /// to this API._ + /// _**Note:** your application is will only be delivered a single + /// [`MainEvent::InputAvailable`] event between calls to this API._ /// - /// To reduce overhead, by default only [`input::Axis::X`] and [`input::Axis::Y`] are enabled + /// To reduce overhead, by default, only [`input::Axis::X`] and [`input::Axis::Y`] are enabled /// and other axis should be enabled explicitly via [`Self::enable_motion_axis`]. - pub fn input_events(&self, callback: F) - where - F: FnMut(&input::InputEvent) -> InputStatus, - { - self.inner.read().unwrap().input_events(callback) + /// + /// This isn't the most ergonomic iteration API since we can't return a standard `Iterator`: + /// - This API returns a lending iterator may borrow from the internal buffer + /// of pending events without copying them. + /// - For each event we want to ensure the application reports whether the + /// event was handled. + /// + /// # Example + /// Code to iterate all pending input events would look something like this: + /// + /// ```rust + /// match app.input_events_iter() { + /// Ok(mut iter) => { + /// loop { + /// let read_input = iter.next(|event| { + /// let handled = match event { + /// InputEvent::KeyEvent(key_event) => { + /// // Snip + /// } + /// InputEvent::MotionEvent(motion_event) => { + /// // Snip + /// } + /// event => { + /// // Snip + /// } + /// }; + /// + /// handled + /// }); + /// + /// if !read_input { + /// break; + /// } + /// } + /// } + /// Err(err) => { + /// log::error!("Failed to get input events iterator: {err:?}"); + /// } + /// } + /// ``` + /// + /// # Panics + /// + /// This must only be called from your `android_main()` thread and it may panic if called + /// from another thread. + pub fn input_events_iter(&self) -> Result { + let receiver = { + let guard = self.inner.read().unwrap(); + guard.input_events_receiver()? + }; + + Ok(input::InputIterator { + inner: receiver.into(), + }) + } + + /// Lookup the [`KeyCharacterMap`] for the given input `device_id` + /// + /// Use [`KeyCharacterMap::get`] to map key codes + meta state into unicode characters + /// or dead keys that compose with the next key. + /// + /// # Example + /// + /// Code to handle unicode character mapping as well as combining dead keys could look some thing like: + /// + /// ```rust + /// let mut combining_accent = None; + /// // Snip + /// + /// let combined_key_char = if let Ok(map) = app.device_key_character_map(device_id) { + /// match map.get(key_event.key_code(), key_event.meta_state()) { + /// Ok(KeyMapChar::Unicode(unicode)) => { + /// let combined_unicode = if let Some(accent) = combining_accent { + /// match map.get_dead_char(accent, unicode) { + /// Ok(Some(key)) => { + /// info!("KeyEvent: Combined '{unicode}' with accent '{accent}' to give '{key}'"); + /// Some(key) + /// } + /// Ok(None) => None, + /// Err(err) => { + /// log::error!("KeyEvent: Failed to combine 'dead key' accent '{accent}' with '{unicode}': {err:?}"); + /// None + /// } + /// } + /// } else { + /// info!("KeyEvent: Pressed '{unicode}'"); + /// Some(unicode) + /// }; + /// combining_accent = None; + /// combined_unicode.map(|unicode| KeyMapChar::Unicode(unicode)) + /// } + /// Ok(KeyMapChar::CombiningAccent(accent)) => { + /// info!("KeyEvent: Pressed 'dead key' combining accent '{accent}'"); + /// combining_accent = Some(accent); + /// Some(KeyMapChar::CombiningAccent(accent)) + /// } + /// Ok(KeyMapChar::None) => { + /// info!("KeyEvent: Pressed non-unicode key"); + /// combining_accent = None; + /// None + /// } + /// Err(err) => { + /// log::error!("KeyEvent: Failed to get key map character: {err:?}"); + /// combining_accent = None; + /// None + /// } + /// } + /// } else { + /// None + /// }; + /// ``` + /// + /// # Errors + /// + /// Since this API needs to use JNI internally to call into the Android JVM it may return + /// a [`error::AppError::JavaError`] in case there is a spurious JNI error or an exception + /// is caught. + pub fn device_key_character_map(&self, device_id: i32) -> Result { + Ok(self + .inner + .read() + .unwrap() + .device_key_character_map(device_id)?) } /// The user-visible SDK version of the framework diff --git a/android-activity/src/native_activity/glue.rs b/android-activity/src/native_activity/glue.rs index 6682857..495c813 100644 --- a/android-activity/src/native_activity/glue.rs +++ b/android-activity/src/native_activity/glue.rs @@ -17,6 +17,7 @@ use log::Level; use ndk::{configuration::Configuration, input_queue::InputQueue, native_window::NativeWindow}; use crate::{ + jni_utils::CloneJavaVM, util::android_log, util::{abort_on_panic, log_panic}, ConfigurationRef, @@ -878,24 +879,21 @@ extern "C" fn ANativeActivity_onCreate( std::thread::spawn(move || { let activity: *mut ndk_sys::ANativeActivity = activity_ptr as *mut _; - let jvm = unsafe { + let jvm = abort_on_panic(|| unsafe { let na = activity; - let jvm = (*na).vm; + let jvm: *mut jni_sys::JavaVM = (*na).vm; let activity = (*na).clazz; // Completely bogus name; this is the _instance_ not class pointer ndk_context::initialize_android_context(jvm.cast(), activity.cast()); + let jvm = CloneJavaVM::from_raw(jvm).unwrap(); // Since this is a newly spawned thread then the JVM hasn't been attached // to the thread yet. Attach before calling the applications main function // so they can safely make JNI calls - let mut jenv_out: *mut core::ffi::c_void = std::ptr::null_mut(); - if let Some(attach_current_thread) = (*(*jvm)).AttachCurrentThread { - attach_current_thread(jvm, &mut jenv_out, std::ptr::null_mut()); - } - + jvm.attach_current_thread_permanently().unwrap(); jvm - }; + }); - let app = AndroidApp::new(rust_glue.clone()); + let app = AndroidApp::new(rust_glue.clone(), jvm.clone()); rust_glue.notify_main_thread_running(); @@ -921,9 +919,9 @@ extern "C" fn ANativeActivity_onCreate( // to the main thread of the process where the Java finish call will take place" ndk_sys::ANativeActivity_finish(activity); - if let Some(detach_current_thread) = (*(*jvm)).DetachCurrentThread { - detach_current_thread(jvm); - } + // This should detach automatically but lets detach explicitly to avoid depending + // on the TLS trickery in `jni-rs` + jvm.detach_current_thread(); ndk_context::release_android_context(); } diff --git a/android-activity/src/native_activity/input.rs b/android-activity/src/native_activity/input.rs index 08db48f..2056fe0 100644 --- a/android-activity/src/native_activity/input.rs +++ b/android-activity/src/native_activity/input.rs @@ -326,6 +326,15 @@ impl<'a> KeyEvent<'a> { pub fn scan_code(&self) -> i32 { self.ndk_event.scan_code() } + + /// Returns the state of the modifiers during this key event, represented by a bitmask. + /// + /// See [the NDK + /// docs](https://developer.android.com/ndk/reference/group/input#akeyevent_getmetastate) + #[inline] + pub fn meta_state(&self) -> MetaState { + self.ndk_event.meta_state() + } } // We use our own wrapper type for input events to have better consistency diff --git a/android-activity/src/native_activity/mod.rs b/android-activity/src/native_activity/mod.rs index 3f16550..fee09b9 100644 --- a/android-activity/src/native_activity/mod.rs +++ b/android-activity/src/native_activity/mod.rs @@ -1,15 +1,22 @@ #![cfg(any(feature = "native-activity", doc))] +use std::collections::HashMap; +use std::marker::PhantomData; +use std::panic::AssertUnwindSafe; use std::ptr; use std::ptr::NonNull; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock, Weak}; use std::time::Duration; use libc::c_void; use log::{error, trace}; +use ndk::input_queue::InputQueue; use ndk::{asset::AssetManager, native_window::NativeWindow}; +use crate::error::{InternalAppError, InternalResult}; +use crate::input::{KeyCharacterMap, KeyCharacterMapBinding}; use crate::input::{TextInputState, TextSpan}; +use crate::jni_utils::{self, CloneJavaVM}; use crate::{ util, AndroidApp, ConfigurationRef, InputStatus, MainEvent, PollEvent, Rect, WindowManagerFlags, }; @@ -79,13 +86,26 @@ impl AndroidAppWaker { } impl AndroidApp { - pub(crate) fn new(native_activity: NativeActivityGlue) -> Self { + pub(crate) fn new(native_activity: NativeActivityGlue, jvm: CloneJavaVM) -> Self { + let mut env = jvm.get_env().unwrap(); // We attach to the thread before creating the AndroidApp + + let key_map_binding = match KeyCharacterMapBinding::new(&mut env) { + Ok(b) => b, + Err(err) => { + panic!("Failed to create KeyCharacterMap JNI bindings: {err:?}"); + } + }; + let app = Self { inner: Arc::new(RwLock::new(AndroidAppInner { + jvm, native_activity, looper: Looper { ptr: ptr::null_mut(), }, + key_map_binding: Arc::new(key_map_binding), + key_maps: Mutex::new(HashMap::new()), + input_receiver: Mutex::new(None), })), }; @@ -122,8 +142,23 @@ unsafe impl Sync for Looper {} #[derive(Debug)] pub(crate) struct AndroidAppInner { + pub(crate) jvm: CloneJavaVM, + pub(crate) native_activity: NativeActivityGlue, looper: Looper, + + /// Shared JNI bindings for the `KeyCharacterMap` class + key_map_binding: Arc, + + /// A table of `KeyCharacterMap`s per `InputDevice` ID + /// these are used to be able to map key presses to unicode + /// characters + key_maps: Mutex>, + + /// While an app is reading input events it holds an + /// InputReceiver reference which we track to ensure + /// we don't hand out more than one receiver at a time + input_receiver: Mutex>>, } impl AndroidAppInner { @@ -356,6 +391,25 @@ impl AndroidAppInner { // NOP: Unsupported } + pub fn device_key_character_map(&self, device_id: i32) -> InternalResult { + let mut guard = self.key_maps.lock().unwrap(); + + let key_map = match guard.entry(device_id) { + std::collections::hash_map::Entry::Occupied(occupied) => occupied.get().clone(), + std::collections::hash_map::Entry::Vacant(vacant) => { + let character_map = jni_utils::device_key_character_map( + self.jvm.clone(), + self.key_map_binding.clone(), + device_id, + )?; + vacant.insert(character_map.clone()); + character_map + } + }; + + Ok(key_map) + } + pub fn enable_motion_axis(&self, _axis: input::Axis) { // NOP - The InputQueue API doesn't let us optimize which axis values are read } @@ -364,10 +418,16 @@ impl AndroidAppInner { // NOP - The InputQueue API doesn't let us optimize which axis values are read } - pub fn input_events(&self, mut callback: F) - where - F: FnMut(&input::InputEvent) -> InputStatus, - { + pub fn input_events_receiver(&self) -> InternalResult> { + let mut guard = self.input_receiver.lock().unwrap(); + + if let Some(receiver) = &*guard { + if receiver.strong_count() > 0 { + return Err(crate::error::InternalAppError::InputUnavailable); + } + } + *guard = None; + // Get the InputQueue for the NativeActivity (if there is one) and also ensure // the queue is re-attached to our event Looper (so new input events will again // trigger a wake up) @@ -376,40 +436,13 @@ impl AndroidAppInner { .looper_attached_input_queue(self.looper(), LOOPER_ID_INPUT); let queue = match queue { Some(queue) => queue, - None => return, + None => return Err(InternalAppError::InputUnavailable), }; - // Note: we basically ignore errors from get_event() currently. Looking - // at the source code for Android's InputQueue, the only error that - // can be returned here is 'WOULD_BLOCK', which we want to just treat as - // meaning the queue is empty. - // - // ref: https://github.com/aosp-mirror/platform_frameworks_base/blob/master/core/jni/android_view_InputQueue.cpp - // - while let Ok(Some(event)) = queue.get_event() { - if let Some(ndk_event) = queue.pre_dispatch(event) { - let event = match ndk_event { - ndk::event::InputEvent::MotionEvent(e) => { - input::InputEvent::MotionEvent(input::MotionEvent::new(e)) - } - ndk::event::InputEvent::KeyEvent(e) => { - input::InputEvent::KeyEvent(input::KeyEvent::new(e)) - } - }; - let handled = callback(&event); + let receiver = Arc::new(InputReceiver { queue }); - let ndk_event = match event { - input::InputEvent::MotionEvent(e) => { - ndk::event::InputEvent::MotionEvent(e.into_ndk_event()) - } - input::InputEvent::KeyEvent(e) => { - ndk::event::InputEvent::KeyEvent(e.into_ndk_event()) - } - _ => unreachable!(), - }; - queue.finish_event(ndk_event, matches!(handled, InputStatus::Handled)); - } - } + *guard = Some(Arc::downgrade(&receiver)); + Ok(receiver) } pub fn internal_data_path(&self) -> Option { @@ -427,3 +460,85 @@ impl AndroidAppInner { unsafe { util::try_get_path_from_ptr((*na).obbPath) } } } + +#[derive(Debug)] +pub(crate) struct InputReceiver { + queue: InputQueue, +} + +impl<'a> From> for InputIteratorInner<'a> { + fn from(receiver: Arc) -> Self { + Self { + receiver, + _lifetime: PhantomData, + } + } +} + +pub(crate) struct InputIteratorInner<'a> { + // Held to maintain exclusive access to buffered input events + receiver: Arc, + _lifetime: PhantomData<&'a ()>, +} + +impl<'a> InputIteratorInner<'a> { + pub(crate) fn next(&self, callback: F) -> bool + where + F: FnOnce(&input::InputEvent) -> InputStatus, + { + // Note: we basically ignore errors from get_event() currently. Looking + // at the source code for Android's InputQueue, the only error that + // can be returned here is 'WOULD_BLOCK', which we want to just treat as + // meaning the queue is empty. + // + // ref: https://github.com/aosp-mirror/platform_frameworks_base/blob/master/core/jni/android_view_InputQueue.cpp + // + if let Ok(Some(ndk_event)) = self.receiver.queue.get_event() { + log::info!("queue: got event: {ndk_event:?}"); + + if let Some(ndk_event) = self.receiver.queue.pre_dispatch(ndk_event) { + let event = match ndk_event { + ndk::event::InputEvent::MotionEvent(e) => { + input::InputEvent::MotionEvent(input::MotionEvent::new(e)) + } + ndk::event::InputEvent::KeyEvent(e) => { + input::InputEvent::KeyEvent(input::KeyEvent::new(e)) + } + }; + + // `finish_event` needs to be called for each event otherwise + // the app would likely get an ANR + let result = std::panic::catch_unwind(AssertUnwindSafe(|| callback(&event))); + + let ndk_event = match event { + input::InputEvent::MotionEvent(e) => { + ndk::event::InputEvent::MotionEvent(e.into_ndk_event()) + } + input::InputEvent::KeyEvent(e) => { + ndk::event::InputEvent::KeyEvent(e.into_ndk_event()) + } + _ => unreachable!(), + }; + + let handled = match result { + Ok(handled) => handled, + Err(payload) => { + log::error!("Calling `finish_event` after panic in input event handler, to try and avoid being killed via an ANR"); + self.receiver.queue.finish_event(ndk_event, false); + std::panic::resume_unwind(payload); + } + }; + + log::info!("queue: finishing event"); + self.receiver + .queue + .finish_event(ndk_event, handled == InputStatus::Handled); + } + + true + } else { + log::info!("queue: no more events"); + false + } + } +} diff --git a/examples/agdk-mainloop/src/lib.rs b/examples/agdk-mainloop/src/lib.rs index 166ee69..a2f94e7 100644 --- a/examples/agdk-mainloop/src/lib.rs +++ b/examples/agdk-mainloop/src/lib.rs @@ -1,4 +1,7 @@ -use android_activity::{AndroidApp, InputStatus, MainEvent, PollEvent}; +use android_activity::{ + input::{InputEvent, KeyAction, KeyEvent, KeyMapChar, MotionAction}, + AndroidApp, InputStatus, MainEvent, PollEvent, +}; use log::info; #[no_mangle] @@ -9,6 +12,8 @@ fn android_main(app: AndroidApp) { let mut redraw_pending = true; let mut native_window: Option = None; + let mut combining_accent = None; + while !quit { app.poll_events( Some(std::time::Duration::from_secs(1)), /* timeout */ @@ -68,11 +73,56 @@ fn android_main(app: AndroidApp) { if let Some(native_window) = &native_window { redraw_pending = false; - // Handle input - app.input_events(|event| { - info!("Input Event: {event:?}"); - InputStatus::Unhandled - }); + // Handle input, via a lending iterator + match app.input_events_iter() { + Ok(mut iter) => loop { + info!("Checking for next input event..."); + if !iter.next(|event| { + match event { + InputEvent::KeyEvent(key_event) => { + let combined_key_char = character_map_and_combine_key( + &app, + key_event, + &mut combining_accent, + ); + info!("KeyEvent: combined key: {combined_key_char:?}") + } + InputEvent::MotionEvent(motion_event) => { + println!("action = {:?}", motion_event.action()); + match motion_event.action() { + MotionAction::Up => { + let pointer = motion_event.pointer_index(); + let pointer = + motion_event.pointer_at_index(pointer); + let x = pointer.x(); + let y = pointer.y(); + + println!("POINTER UP {x}, {y}"); + if x < 200.0 && y < 200.0 { + println!("Requesting to show keyboard"); + app.show_soft_input(true); + } + } + _ => {} + } + } + InputEvent::TextEvent(state) => { + info!("Input Method State: {state:?}"); + } + _ => {} + } + + info!("Input Event: {event:?}"); + InputStatus::Unhandled + }) { + info!("No more input available"); + break; + } + }, + Err(err) => { + log::error!("Failed to get input events iterator: {err:?}"); + } + } info!("Render..."); dummy_render(native_window); @@ -83,6 +133,73 @@ fn android_main(app: AndroidApp) { } } +/// Tries to map the `key_event` to a `KeyMapChar` containing a unicode character or dead key accent +/// +/// This shows how to take a `KeyEvent` and look up its corresponding `KeyCharacterMap` and +/// use that to try and map the `key_code` + `meta_state` to a unicode character or a +/// dead key that be combined with the next key press. +fn character_map_and_combine_key( + app: &AndroidApp, + key_event: &KeyEvent, + combining_accent: &mut Option, +) -> Option { + let device_id = key_event.device_id(); + + let key_map = match app.device_key_character_map(device_id) { + Ok(key_map) => key_map, + Err(err) => { + log::error!("Failed to look up `KeyCharacterMap` for device {device_id}: {err:?}"); + return None; + } + }; + + match key_map.get(key_event.key_code(), key_event.meta_state()) { + Ok(KeyMapChar::Unicode(unicode)) => { + // Only do dead key combining on key down + if key_event.action() == KeyAction::Down { + let combined_unicode = if let Some(accent) = combining_accent { + match key_map.get_dead_char(*accent, unicode) { + Ok(Some(key)) => { + info!("KeyEvent: Combined '{unicode}' with accent '{accent}' to give '{key}'"); + Some(key) + } + Ok(None) => None, + Err(err) => { + log::error!("KeyEvent: Failed to combine 'dead key' accent '{accent}' with '{unicode}': {err:?}"); + None + } + } + } else { + info!("KeyEvent: Pressed '{unicode}'"); + Some(unicode) + }; + *combining_accent = None; + combined_unicode.map(|unicode| KeyMapChar::Unicode(unicode)) + } else { + Some(KeyMapChar::Unicode(unicode)) + } + } + Ok(KeyMapChar::CombiningAccent(accent)) => { + if key_event.action() == KeyAction::Down { + info!("KeyEvent: Pressed 'dead key' combining accent '{accent}'"); + *combining_accent = Some(accent); + } + Some(KeyMapChar::CombiningAccent(accent)) + } + Ok(KeyMapChar::None) => { + // Leave any combining_accent state in tact (seems to match how other + // Android apps work) + info!("KeyEvent: Pressed non-unicode key"); + None + } + Err(err) => { + log::error!("KeyEvent: Failed to get key map character: {err:?}"); + *combining_accent = None; + None + } + } +} + /// Post a NOP frame to the window /// /// Since this is a bare minimum test app we don't depend diff --git a/examples/na-mainloop/src/lib.rs b/examples/na-mainloop/src/lib.rs index 166ee69..a2f94e7 100644 --- a/examples/na-mainloop/src/lib.rs +++ b/examples/na-mainloop/src/lib.rs @@ -1,4 +1,7 @@ -use android_activity::{AndroidApp, InputStatus, MainEvent, PollEvent}; +use android_activity::{ + input::{InputEvent, KeyAction, KeyEvent, KeyMapChar, MotionAction}, + AndroidApp, InputStatus, MainEvent, PollEvent, +}; use log::info; #[no_mangle] @@ -9,6 +12,8 @@ fn android_main(app: AndroidApp) { let mut redraw_pending = true; let mut native_window: Option = None; + let mut combining_accent = None; + while !quit { app.poll_events( Some(std::time::Duration::from_secs(1)), /* timeout */ @@ -68,11 +73,56 @@ fn android_main(app: AndroidApp) { if let Some(native_window) = &native_window { redraw_pending = false; - // Handle input - app.input_events(|event| { - info!("Input Event: {event:?}"); - InputStatus::Unhandled - }); + // Handle input, via a lending iterator + match app.input_events_iter() { + Ok(mut iter) => loop { + info!("Checking for next input event..."); + if !iter.next(|event| { + match event { + InputEvent::KeyEvent(key_event) => { + let combined_key_char = character_map_and_combine_key( + &app, + key_event, + &mut combining_accent, + ); + info!("KeyEvent: combined key: {combined_key_char:?}") + } + InputEvent::MotionEvent(motion_event) => { + println!("action = {:?}", motion_event.action()); + match motion_event.action() { + MotionAction::Up => { + let pointer = motion_event.pointer_index(); + let pointer = + motion_event.pointer_at_index(pointer); + let x = pointer.x(); + let y = pointer.y(); + + println!("POINTER UP {x}, {y}"); + if x < 200.0 && y < 200.0 { + println!("Requesting to show keyboard"); + app.show_soft_input(true); + } + } + _ => {} + } + } + InputEvent::TextEvent(state) => { + info!("Input Method State: {state:?}"); + } + _ => {} + } + + info!("Input Event: {event:?}"); + InputStatus::Unhandled + }) { + info!("No more input available"); + break; + } + }, + Err(err) => { + log::error!("Failed to get input events iterator: {err:?}"); + } + } info!("Render..."); dummy_render(native_window); @@ -83,6 +133,73 @@ fn android_main(app: AndroidApp) { } } +/// Tries to map the `key_event` to a `KeyMapChar` containing a unicode character or dead key accent +/// +/// This shows how to take a `KeyEvent` and look up its corresponding `KeyCharacterMap` and +/// use that to try and map the `key_code` + `meta_state` to a unicode character or a +/// dead key that be combined with the next key press. +fn character_map_and_combine_key( + app: &AndroidApp, + key_event: &KeyEvent, + combining_accent: &mut Option, +) -> Option { + let device_id = key_event.device_id(); + + let key_map = match app.device_key_character_map(device_id) { + Ok(key_map) => key_map, + Err(err) => { + log::error!("Failed to look up `KeyCharacterMap` for device {device_id}: {err:?}"); + return None; + } + }; + + match key_map.get(key_event.key_code(), key_event.meta_state()) { + Ok(KeyMapChar::Unicode(unicode)) => { + // Only do dead key combining on key down + if key_event.action() == KeyAction::Down { + let combined_unicode = if let Some(accent) = combining_accent { + match key_map.get_dead_char(*accent, unicode) { + Ok(Some(key)) => { + info!("KeyEvent: Combined '{unicode}' with accent '{accent}' to give '{key}'"); + Some(key) + } + Ok(None) => None, + Err(err) => { + log::error!("KeyEvent: Failed to combine 'dead key' accent '{accent}' with '{unicode}': {err:?}"); + None + } + } + } else { + info!("KeyEvent: Pressed '{unicode}'"); + Some(unicode) + }; + *combining_accent = None; + combined_unicode.map(|unicode| KeyMapChar::Unicode(unicode)) + } else { + Some(KeyMapChar::Unicode(unicode)) + } + } + Ok(KeyMapChar::CombiningAccent(accent)) => { + if key_event.action() == KeyAction::Down { + info!("KeyEvent: Pressed 'dead key' combining accent '{accent}'"); + *combining_accent = Some(accent); + } + Some(KeyMapChar::CombiningAccent(accent)) + } + Ok(KeyMapChar::None) => { + // Leave any combining_accent state in tact (seems to match how other + // Android apps work) + info!("KeyEvent: Pressed non-unicode key"); + None + } + Err(err) => { + log::error!("KeyEvent: Failed to get key map character: {err:?}"); + *combining_accent = None; + None + } + } +} + /// Post a NOP frame to the window /// /// Since this is a bare minimum test app we don't depend