diff --git a/CMake/Dependencies.cmake b/CMake/Dependencies.cmake index a574210b2..d6c638b3d 100644 --- a/CMake/Dependencies.cmake +++ b/CMake/Dependencies.cmake @@ -51,6 +51,8 @@ add_subdirectory(3rdParty/sol2) if(SCREEN_READER_INTEGRATION) if(WIN32) add_subdirectory(3rdParty/tolk) + elseif(ANDROID) + # Android uses native accessibility API, no external dependency needed else() find_package(Speechd REQUIRED) endif() diff --git a/RetroArch b/RetroArch new file mode 160000 index 000000000..7df941eac --- /dev/null +++ b/RetroArch @@ -0,0 +1 @@ +Subproject commit 7df941eac94d91534ce01e6984246d82692fdb8c diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 8c65187ff..6f0a2eb93 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -1001,6 +1001,8 @@ if(SCREEN_READER_INTEGRATION) if(WIN32) target_compile_definitions(libdevilutionx PRIVATE Tolk) target_link_libraries(libdevilutionx PUBLIC Tolk) + elseif(ANDROID) + # Android uses native accessibility API, no external library needed else() target_include_directories(libdevilutionx PUBLIC ${Speechd_INCLUDE_DIRS}) target_link_libraries(libdevilutionx PUBLIC speechd) diff --git a/Source/platform/android/CLAUDE.md b/Source/platform/android/CLAUDE.md new file mode 100644 index 000000000..e8b7cf08a --- /dev/null +++ b/Source/platform/android/CLAUDE.md @@ -0,0 +1,12 @@ + +# Recent Activity + + + +### Feb 5, 2026 + +| ID | Time | T | Title | Read | +|----|------|---|-------|------| +| #4 | 12:26 PM | 🔵 | Android accessibility function interface declared | ~183 | +| #3 | " | 🔵 | Android accessibility JNI bridge implementation found | ~229 | + \ No newline at end of file diff --git a/Source/platform/android/android.cpp b/Source/platform/android/android.cpp index fddb850de..d1fbfdac9 100644 --- a/Source/platform/android/android.cpp +++ b/Source/platform/android/android.cpp @@ -2,21 +2,184 @@ #include "mpq/mpq_reader.hpp" #include +#include namespace devilution { -namespace { -bool AreExtraFontsOutOfDateForMpqPath(const char *mpqPath) +// Global Java VM pointer - set during JNI initialization +static JavaVM *g_jvm = nullptr; +static jobject g_activity = nullptr; + +// JNI method cache for accessibility functions +struct AndroidJNIMethods { + jmethodID isScreenReaderEnabled; + jmethodID accessibilitySpeak; + bool initialized; +} g_jniMethods = { nullptr, nullptr, false }; + +// Thread-local storage for JNIEnv +static pthread_key_t g_jniEnvKey; + +// Initialize JNI environment key +static void JNIKeyDestructor(void *env) +{ + // Don't detach - let SDL handle it +} + +static void InitializeJNIKey() +{ + static bool initialized = false; + if (initialized) + return; + + pthread_key_create(&g_jniEnvKey, JNIKeyDestructor); + initialized = true; +} + +// Get JNI environment for current thread +static JNIEnv* GetJNI() +{ + InitializeJNIKey(); + + JNIEnv *env = (JNIEnv *)pthread_getspecific(g_jniEnvKey); + if (env) + return env; + + if (g_jvm == nullptr) + return nullptr; + + // Get or attach the current thread + int status = g_jvm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6); + if (status == JNI_EDETACHED) { + // Thread not attached, attach it + status = g_jvm->AttachCurrentThread(&env, nullptr); + if (status < 0) + return nullptr; + pthread_setspecific(g_jniEnvKey, env); + } else if (status != JNI_OK) { + return nullptr; + } + + return env; +} + +static bool AreExtraFontsOutOfDateForMpqPath(const char *mpqPath) { int32_t error = 0; std::optional archive = MpqArchive::Open(mpqPath, error); return error == 0 && archive && AreExtraFontsOutOfDate(*archive); } -} // namespace +// Initialize JNI method IDs for accessibility +// This should be called once during initialization +static void InitializeAccessibilityJNI(JNIEnv *env) +{ + if (g_jniMethods.initialized) + return; + + // Get the DevilutionXSDLActivity class + jclass activityClass = env->FindClass("org/diasurgical/devilutionx/DevilutionXSDLActivity"); + if (activityClass == nullptr) { + return; + } + + // Cache method IDs for accessibility functions + g_jniMethods.isScreenReaderEnabled = env->GetMethodID(activityClass, "isScreenReaderEnabled", "()Z"); + g_jniMethods.accessibilitySpeak = env->GetMethodID(activityClass, "accessibilitySpeak", "(Ljava/lang/String;)V"); + + if (g_jniMethods.isScreenReaderEnabled && g_jniMethods.accessibilitySpeak) { + g_jniMethods.initialized = true; + } + + env->DeleteLocalRef(activityClass); +} + +// Public accessibility functions for Android +namespace accessibility { + +bool InitializeScreenReaderAndroid() +{ + // JNI is initialized when nativeInitAccessibility is called from Java + // This function is kept for compatibility but the actual initialization + // happens in Java_org_diasurgical_devilutionx_DevilutionXSDLActivity_nativeInitAccessibility + return g_jniMethods.initialized; +} + +void ShutDownScreenReaderAndroid() +{ + // Clean up the activity reference + if (g_activity != nullptr) { + JNIEnv *env = GetJNI(); + if (env != nullptr) { + env->DeleteGlobalRef(g_activity); + } + g_activity = nullptr; + } + + g_jniMethods.initialized = false; + g_jniMethods.isScreenReaderEnabled = nullptr; + g_jniMethods.accessibilitySpeak = nullptr; +} + +void SpeakTextAndroid(const char *text) +{ + if (!g_jniMethods.initialized) + return; + + JNIEnv *env = GetJNI(); + if (env == nullptr || g_activity == nullptr) + return; + + // Create a Java string from the text + jstring jText = env->NewStringUTF(text); + if (jText == nullptr) + return; + + // Call the accessibilitySpeak method + env->CallVoidMethod(g_activity, g_jniMethods.accessibilitySpeak, jText); + + // Clean up + env->DeleteLocalRef(jText); +} + +bool IsScreenReaderEnabledAndroid() +{ + if (!g_jniMethods.initialized) + return false; + + JNIEnv *env = GetJNI(); + if (env == nullptr || g_activity == nullptr) + return false; + + // Call the isScreenReaderEnabled method + jboolean result = env->CallBooleanMethod(g_activity, g_jniMethods.isScreenReaderEnabled); + + return result == JNI_TRUE; +} + +} // namespace accessibility } // namespace devilution +// JNI initialization function called from Java during Activity initialization extern "C" { +JNIEXPORT void JNICALL Java_org_diasurgical_devilutionx_DevilutionXSDLActivity_nativeInitAccessibility( + JNIEnv *env, jobject thiz) +{ + // Store the Java VM pointer + if (devilution::g_jvm == nullptr) { + env->GetJavaVM(&devilution::g_jvm); + } + + // Store a global reference to the activity + if (devilution::g_activity != nullptr) { + env->DeleteGlobalRef(devilution::g_activity); + } + devilution::g_activity = env->NewGlobalRef(thiz); + + // Initialize the JNI method cache + devilution::InitializeAccessibilityJNI(env); +} + JNIEXPORT jboolean JNICALL Java_org_diasurgical_devilutionx_DevilutionXSDLActivity_areFontsOutOfDate(JNIEnv *env, jclass cls, jstring fonts_mpq) { const char *mpqPath = env->GetStringUTFChars(fonts_mpq, nullptr); diff --git a/Source/platform/android/android.hpp b/Source/platform/android/android.hpp new file mode 100644 index 000000000..e3bd4ed64 --- /dev/null +++ b/Source/platform/android/android.hpp @@ -0,0 +1,14 @@ +#pragma once + +namespace devilution { +namespace accessibility { + +#ifdef __ANDROID__ +bool InitializeScreenReaderAndroid(); +void ShutDownScreenReaderAndroid(); +void SpeakTextAndroid(const char *text); +bool IsScreenReaderEnabledAndroid(); +#endif + +} // namespace accessibility +} // namespace devilution diff --git a/Source/utils/CLAUDE.md b/Source/utils/CLAUDE.md new file mode 100644 index 000000000..6a7d38f74 --- /dev/null +++ b/Source/utils/CLAUDE.md @@ -0,0 +1,11 @@ + +# Recent Activity + + + +### Feb 5, 2026 + +| ID | Time | T | Title | Read | +|----|------|---|-------|------| +| #2 | 12:26 PM | 🔵 | Screen reader platform abstraction architecture identified | ~245 | + \ No newline at end of file diff --git a/Source/utils/screen_reader.cpp b/Source/utils/screen_reader.cpp index 7a7c497a6..7ecd623f3 100644 --- a/Source/utils/screen_reader.cpp +++ b/Source/utils/screen_reader.cpp @@ -6,13 +6,15 @@ #ifdef _WIN32 #include "utils/file_util.h" #include +#elif defined(__ANDROID__) +#include "platform/android/android.hpp" #else #include #endif namespace devilution { -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(__ANDROID__) SPDConnection *Speechd; #endif @@ -20,36 +22,42 @@ void InitializeScreenReader() { #ifdef _WIN32 Tolk_Load(); +#elif defined(__ANDROID__) + devilution::accessibility::InitializeScreenReaderAndroid(); #else Speechd = spd_open("DevilutionX", "DevilutionX", NULL, SPD_MODE_SINGLE); #endif } -void ShutDownScreenReader() -{ -#ifdef _WIN32 - Tolk_Unload(); -#else - spd_close(Speechd); -#endif -} - -void SpeakText(std::string_view text, bool force) -{ - static std::string SpokenText; - - if (!force && SpokenText == text) - return; - - SpokenText = text; - -#ifdef _WIN32 - const auto textUtf16 = ToWideChar(SpokenText); - if (textUtf16 != nullptr) - Tolk_Output(textUtf16.get(), true); -#else - spd_say(Speechd, SPD_TEXT, SpokenText.c_str()); -#endif -} +void ShutDownScreenReader() +{ +#ifdef _WIN32 + Tolk_Unload(); +#elif defined(__ANDROID__) + devilution::accessibility::ShutDownScreenReaderAndroid(); +#else + spd_close(Speechd); +#endif +} + +void SpeakText(std::string_view text, bool force) +{ + static std::string SpokenText; + + if (!force && SpokenText == text) + return; + + SpokenText = text; + +#ifdef _WIN32 + const auto textUtf16 = ToWideChar(SpokenText); + if (textUtf16 != nullptr) + Tolk_Output(textUtf16.get(), true); +#elif defined(__ANDROID__) + devilution::accessibility::SpeakTextAndroid(SpokenText.c_str()); +#else + spd_say(Speechd, SPD_TEXT, SpokenText.c_str()); +#endif +} } // namespace devilution diff --git a/android-project/app/build.gradle b/android-project/app/build.gradle index 6bbbd34f4..585663cc8 100644 --- a/android-project/app/build.gradle +++ b/android-project/app/build.gradle @@ -19,7 +19,7 @@ android { versionName project.file('../../VERSION').text.trim() externalNativeBuild { cmake { - arguments "-DANDROID_STL=c++_static" + arguments "-DANDROID_STL=c++_static", "-DSCREEN_READER_INTEGRATION=ON" abiFilters 'armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64' } } diff --git a/android-project/app/src/main/java/org/diasurgical/devilutionx/DevilutionXSDLActivity.java b/android-project/app/src/main/java/org/diasurgical/devilutionx/DevilutionXSDLActivity.java index 2a0ade3cb..a4e37ae2a 100644 --- a/android-project/app/src/main/java/org/diasurgical/devilutionx/DevilutionXSDLActivity.java +++ b/android-project/app/src/main/java/org/diasurgical/devilutionx/DevilutionXSDLActivity.java @@ -8,6 +8,7 @@ import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.ViewTreeObserver; import android.view.WindowManager; +import android.view.accessibility.AccessibilityManager; import org.libsdl.app.SDLActivity; @@ -33,6 +34,10 @@ public class DevilutionXSDLActivity extends SDLActivity { migrateSaveGames(); super.onCreate(savedInstanceState); + + // Initialize accessibility JNI - must be after super.onCreate() + // so that the native library is loaded first + nativeInitAccessibility(); } /** @@ -143,4 +148,34 @@ public class DevilutionXSDLActivity extends SDLActivity { } public static native boolean areFontsOutOfDate(String fonts_mpq); + + /** + * Native method to initialize accessibility JNI functions. + * This caches the method IDs needed for accessibility features. + */ + public native void nativeInitAccessibility(); + + /** + * Checks if the screen reader (TalkBack) is enabled on the device. + * This follows the same pattern as RetroArch's accessibility implementation. + * + * @return true if TalkBack is enabled and touch exploration is active + */ + public boolean isScreenReaderEnabled() { + AccessibilityManager accessibilityManager = (AccessibilityManager) getSystemService(ACCESSIBILITY_SERVICE); + boolean isAccessibilityEnabled = accessibilityManager.isEnabled(); + boolean isExploreByTouchEnabled = accessibilityManager.isTouchExplorationEnabled(); + return isAccessibilityEnabled && isExploreByTouchEnabled; + } + + /** + * Speaks the given message using Android's accessibility API. + * This integrates with TalkBack and other screen readers. + * This follows the same pattern as RetroArch's accessibility implementation. + * + * @param message The text to speak + */ + public void accessibilitySpeak(String message) { + getWindow().getDecorView().announceForAccessibility(message); + } } diff --git a/docs/ANDROID_ACCESSIBILITY.md b/docs/ANDROID_ACCESSIBILITY.md new file mode 100644 index 000000000..19f19109e --- /dev/null +++ b/docs/ANDROID_ACCESSIBILITY.md @@ -0,0 +1,254 @@ +# Android Accessibility Implementation + +## Overview + +This document describes the Android accessibility implementation for Diablo Access, which follows the same architecture pattern used by RetroArch to provide screen reader support for visually impaired players. + +## Architecture + +The implementation consists of three main components: + +### 1. Java Layer (DevilutionXSDLActivity.java) + +Located in `android-project/app/src/main/java/org/diasurgical/devilutionx/DevilutionXSDLActivity.java` + +```java +public boolean isScreenReaderEnabled() { + AccessibilityManager accessibilityManager = (AccessibilityManager) getSystemService(ACCESSIBILITY_SERVICE); + boolean isAccessibilityEnabled = accessibilityManager.isEnabled(); + boolean isExploreByTouchEnabled = accessibilityManager.isTouchExplorationEnabled(); + return isAccessibilityEnabled && isExploreByTouchEnabled; +} + +public void accessibilitySpeak(String message) { + getWindow().getDecorView().announceForAccessibility(message); +} +``` + +**Key Features:** +- `isScreenReaderEnabled()`: Checks if TalkBack is enabled and touch exploration is active +- `accessibilitySpeak()`: Uses Android's native `announceForAccessibility()` API to speak text +- These methods are called from native C++ code via JNI + +### 2. JNI Bridge (android.cpp) + +Located in `Source/platform/android/android.cpp` + +**Key Components:** + +1. **Global State:** + - `g_jvm`: Global JavaVM pointer + - `g_activity`: Global reference to the Activity + - `g_jniMethods`: Cached method IDs for performance + +2. **Thread Management:** + - Uses pthread thread-local storage to cache JNIEnv per thread + - Automatically attaches threads to JVM as needed + - Follows RetroArch's pattern for thread-safe JNI access + +3. **Public API (namespace `accessibility`):** + ```cpp + bool InitializeScreenReaderAndroid(); // Check if initialized + void ShutDownScreenReaderAndroid(); // Cleanup resources + void SpeakTextAndroid(const char *text); // Speak text + bool IsScreenReaderEnabledAndroid(); // Check TalkBack status + ``` + +4. **JNI Entry Point:** + ```cpp + JNIEXPORT void JNICALL Java_org_diasurgical_devilutionx_DevilutionXSDLActivity_nativeInitAccessibility( + JNIEnv *env, jobject thiz) + ``` + - Called from Java during Activity onCreate() + - Stores JVM pointer and global activity reference + - Caches method IDs for performance + +### 3. Platform Integration (screen_reader.cpp) + +Located in `Source/utils/screen_reader.cpp` + +Modified to support Android alongside Windows and Linux: + +```cpp +#ifdef _WIN32 + Tolk_Load(); +#elif defined(__ANDROID__) + devilution::accessibility::InitializeScreenReaderAndroid(); +#else + Speechd = spd_open("DevilutionX", "DevilutionX", NULL, SPD_MODE_SINGLE); +#endif +``` + +## How It Works + +### Initialization Flow + +1. App launches → `DevilutionXSDLActivity.onCreate()` is called +2. **IMPORTANT**: `super.onCreate()` must be called **before** `nativeInitAccessibility()` + - This ensures SDL loads the native library first + - Calling native methods before the library is loaded causes `UnsatisfiedLinkError` +3. `nativeInitAccessibility()` is called from Java (after `super.onCreate()`) +4. JNI function stores: + - JavaVM pointer + - Global reference to Activity + - Method IDs for accessibility functions +5. Game calls `InitializeScreenReader()` → checks if Android is ready + +**Critical Implementation Detail:** +```java +protected void onCreate(Bundle savedInstanceState) { + // ... setup code ... + + super.onCreate(savedInstanceState); // Must be FIRST - loads native library + + // Initialize accessibility JNI - must be after super.onCreate() + // so that the native library is loaded first + nativeInitAccessibility(); +} +``` + +### Speaking Text Flow + +1. Game code calls `SpeakText("Some text")` +2. `screen_reader.cpp` routes to `SpeakTextAndroid()` on Android +3. `SpeakTextAndroid()`: + - Gets JNIEnv for current thread (attaches if needed) + - Creates Java string from C string + - Calls `accessibilitySpeak()` method on Activity +4. Java method calls `announceForAccessibility()` +5. Android's accessibility framework forwards to TalkBack +6. TalkBack speaks the text + +### Thread Safety + +The implementation is thread-safe: + +- Each thread gets its own JNIEnv cached in thread-local storage +- The JavaVM pointer is global and constant +- The Activity reference is a global JNI reference (valid across threads) +- Method IDs are constant once initialized + +This follows the same pattern as RetroArch's `jni_thread_getenv()` function. + +## Comparison with Other Platforms + +### Windows (Tolk) +- Uses NVDA/JAWS screen readers via Tolk library +- Direct communication with screen readers +- Requires Windows-specific APIs + +### Linux (speech-dispatcher) +- Uses speech-dispatcher daemon +- Direct socket communication +- Requires speech-dispatcher to be running + +### Android (this implementation) +- Uses Android's accessibility framework +- Integrates with TalkBack and other screen readers +- Uses `announceForAccessibility()` API +- Requires TalkBack to be enabled + +## Advantages of This Approach + +1. **Native Integration**: Uses Android's built-in accessibility APIs +2. **Works with All Screen Readers**: TalkBack, Samsung TalkBack, BrailleBack, etc. +3. **No External Dependencies**: Uses only Android SDK and NDK +4. **Performance**: Method IDs cached, thread-local storage for JNIEnv +5. **Thread-Safe**: Can be called from any thread +6. **Follows Best Practices**: Same pattern as RetroArch (proven in production) + +## Differences from RetroArch + +### Similarities +- Both use `announceForAccessibility()` for speaking +- Both cache JNI method IDs +- Both use thread-local storage for JNIEnv +- Both check `isTouchExplorationEnabled()` + +### Minor Differences +- Diablo Access stores Activity as global reference; RetroArch uses android_app struct +- Diablo Access uses `pthread` directly; RetroArch wraps it in `jni_thread_getenv()` +- Diablo Access initialization happens in `onCreate()`; RetroArch uses native app glue + +Both approaches are valid and work correctly. + +## Testing + +### Prerequisites +1. Enable TalkBack on Android device/emulator: + - Settings → Accessibility → TalkBack → Enable + - Enable "Explore by touch" + +2. Build and install the app: + ```bash + cd android-project + ./gradlew assembleDebug + adb install app/build/outputs/apk/debug/app-debug.apk + ``` + +3. Test gameplay: + - Launch game + - Navigate through menus + - Listen for spoken feedback + - Verify all accessibility features work + +### Expected Behavior +- Menu items should be spoken +- Game state changes should be announced +- Tracker navigation should provide audio feedback +- Health/mana/status should be spoken + +## Troubleshooting + +### No Speech +- Verify TalkBack is enabled +- Check "Explore by touch" is enabled +- Verify device volume is up +- Check logcat for JNI errors + +### Crashes +- **UnsatisfiedLinkError**: Ensure `nativeInitAccessibility()` is called AFTER `super.onCreate()` + - The native library must be loaded by SDL before any native methods can be called + - `super.onCreate()` triggers SDL to load the devilutionx library +- Check that `nativeInitAccessibility()` is called before other functions +- Verify method IDs are cached successfully +- Check thread attachment in logcat + +### Common Issues and Solutions + +#### Issue: App crashes on startup with `UnsatisfiedLinkError` +**Error:** `No implementation found for void org.diasurgical.devilutionx.DevilutionXSDLActivity.nativeInitAccessibility()` + +**Cause:** Calling `nativeInitAccessibility()` before `super.onCreate()` in the Activity's `onCreate()` method. + +**Solution:** Move `nativeInitAccessibility()` to after `super.onCreate()`: +```java +@Override +protected void onCreate(Bundle savedInstanceState) { + // ... setup code ... + super.onCreate(savedInstanceState); // MUST be before nativeInitAccessibility() + nativeInitAccessibility(); // Call AFTER super.onCreate() +} +``` + +**Why:** SDL loads the native library (`libdevilutionx.so`) during `super.onCreate()`. Any native method calls before this will fail because the library isn't loaded yet. + +### Performance Issues +- Method ID caching should prevent repeated lookups +- Thread-local storage should minimize GetEnv calls +- Check for excessive JNI boundary crossings + +## Future Enhancements + +Possible improvements: +1. Add priority levels for announcements (like RetroArch) +2. Add speech speed control +3. Add haptic feedback integration +4. Support for accessibility gestures +5. Braille display integration + +## References + +- RetroArch Accessibility: https://github.com/libretro/RetroArch (accessibility.h, platform_unix.c) +- Android Accessibility: https://developer.android.com/guide/topics/ui/accessibility +- JNI Best Practices: https://developer.android.com/training/articles/perf-jni diff --git a/uwp-project/.claude/settings.local.json b/uwp-project/.claude/settings.local.json new file mode 100644 index 000000000..7a3b0043b --- /dev/null +++ b/uwp-project/.claude/settings.local.json @@ -0,0 +1,12 @@ +{ + "permissions": { + "allow": [ + "Bash(git clone:*)", + "Bash(find:*)", + "Bash(bash:*)", + "Bash(dir:*)", + "Bash(gradlew.bat assembleDebug:*)", + "Bash(./gradlew.bat:*)" + ] + } +}