Compare commits

..
Author SHA1 Message Date
Bailey Dixon 7ae95915bf chore(release): reconcile Server 1.5.0 main release 2026-08-02 18:58:33 -04:00
Bailey Dixon c85f42c51c Merge pull request #279 from Codename-11/dev
release(server): server-v1.5.0
2026-08-02 18:58:20 -04:00
Bailey Dixon 15e0106648 fix(android): preserve legacy collection compatibility 2026-08-02 18:55:21 -04:00
Bailey Dixon 58d7e8581b chore(release): reconcile dev before Android 1.6.0
# Conflicts:
#	CHANGELOG.md
#	DEVLOG.md
2026-08-02 18:51:59 -04:00
Bailey Dixon 8c570b214d Merge pull request #281 from Codename-11/fix/mobile-plugin-path-containment
fix(plugins): contain generated page paths
2026-08-02 18:42:04 -04:00
Bailey Dixon 52f19ca028 fix(plugins): contain generated page paths 2026-08-02 18:40:47 -04:00
Bailey Dixon acc7daa6f2 fix(dev): restore UI preview compilation 2026-08-02 18:29:45 -04:00
Bailey Dixon 9ca42e3e6c Merge pull request #280 from Codename-11/chore/reconcile-main-before-server-1.5.0
chore: reconcile main release history into dev
2026-08-02 18:28:11 -04:00
Bailey Dixon 781969d782 chore(release): reconcile Server 1.5.0 into Android 1.6.0
# Conflicts:
#	CHANGELOG.md
2026-08-02 18:19:47 -04:00
Bailey Dixon 279b83a77c release(android): prepare android-v1.6.0 2026-08-02 18:19:17 -04:00
Bailey Dixon 2f949c7d15 chore: reconcile main release history into dev 2026-08-02 18:17:50 -04:00
Bailey Dixon c77fd057bb Merge pull request #278 from Codename-11/release/server-1.5.0
release(server): server-v1.5.0
2026-08-02 18:16:25 -04:00
Bailey Dixon ed1c47f47d release(server): server-v1.5.0 2026-08-02 18:14:51 -04:00
Bailey Dixon c1800a3ddd chore(release): reconcile dev before Android 1.6.0
# Conflicts:
#	CHANGELOG.md
#	DEVLOG.md
#	app/src/main/kotlin/com/hermesandroid/relay/ui/components/ThinkingBlock.kt
#	app/src/main/kotlin/com/hermesandroid/relay/voice/VoiceOverlayHost.kt
#	app/src/main/res/values-b+pt+BR/strings.xml
#	app/src/main/res/values-b+zh+Hans/strings.xml
#	app/src/main/res/values-de/strings.xml
#	app/src/main/res/values-es/strings.xml
#	app/src/main/res/values-ja/strings.xml
#	app/src/main/res/values/strings.xml
#	docs/localization-status.json
2026-08-02 18:14:45 -04:00
Bailey Dixon f5c2a2b888 feat(plugins): add live Android plugin surfaces 2026-08-02 18:14:42 -04:00
Bailey Dixon 3ec5a09885 chore(release): reconcile main before Android 1.6.0
# Conflicts:
#	DEVLOG.md
#	app/src/main/kotlin/com/hermesandroid/relay/ui/components/VoiceModeOverlay.kt
#	app/src/test/kotlin/com/hermesandroid/relay/ui/components/VoiceModeOverlayStateTest.kt
2026-08-02 18:09:37 -04:00
Bailey Dixon 2a8038a1bd Merge fix/android-pet-mvp-hardening into dev 2026-08-02 14:08:37 -04:00
Bailey Dixon d0684dd252 fix(android): harden floating pet mvp 2026-08-02 14:08:18 -04:00
Bailey Dixon dae75ebe13 feat(android): improve pet route inspector 2026-08-02 13:36:20 -04:00
Bailey Dixon 82fb46b914 Merge fix/android-list-identity-dev into dev 2026-08-02 13:17:30 -04:00
Bailey Dixon 3ac74404bf fix(android): keep pet inspector below app header 2026-08-02 13:12:04 -04:00
Bailey Dixon 9187d8e77c Merge pull request #277 from Codename-11/feature/russian-localization-salvage
feat(android): add Russian localization
2026-08-02 13:11:28 -04:00
Bailey Dixon 9b7bf0f7fd fix(android): preserve streamed list identity 2026-08-02 13:09:20 -04:00
Bailey Dixon a59b54c600 feat(android): collapse pet path inspector by default 2026-08-02 13:07:54 -04:00
Bailey Dixon 9c56598438 feat(android): improve pet terrain inspector 2026-08-02 13:03:25 -04:00
Bailey DixonandDScoNOIZ 4efc52dd5b feat(android): add Russian localization
Salvaged from #276 by @DScoNOIZ.

Co-authored-by: DScoNOIZ <212546794+DScoNOIZ@users.noreply.github.com>
2026-08-02 13:02:57 -04:00
Bailey Dixon b274e6f2a6 feat(android): expand floating pet terrain roaming 2026-08-02 12:28:54 -04:00
Bailey Dixon 6f5c49be17 Merge feature/android-live-plugins into dev
# Conflicts:
#	docs/localization-status.json
2026-08-02 11:39:59 -04:00
Bailey Dixon b328370d32 feat(plugins): add live Android plugin surfaces 2026-08-02 11:38:18 -04:00
Bailey Dixon 0e1d70e8ff fix(android): recover pets from occupied terrain 2026-08-02 10:53:22 -04:00
Bailey Dixon 4ea41386ee fix(android): constrain pet terrain routes 2026-08-02 10:46:59 -04:00
Bailey Dixon 6ac7e5457f feat(android): distinguish possible pet routes 2026-08-02 10:12:08 -04:00
Bailey Dixon 46463f4b00 feat(android): use narrow bubbles as pet hop points 2026-08-02 09:54:12 -04:00
Bailey Dixon 2f72c5444c feat(android): visualize and explore pet terrain 2026-08-02 09:28:09 -04:00
Bailey Dixon ecc97416fc feat(android): add bounded pet terrain journeys 2026-08-02 08:40:27 -04:00
Bailey Dixon 88667eea89 fix(android): hop from measured bubble edges 2026-08-02 07:03:49 -04:00
Bailey Dixon febd938651 fix(android): escape pet from invalid bubble overlap 2026-08-01 22:41:41 -04:00
Bailey Dixon 9419615068 fix(android): recover pet from scrolling chat bubbles 2026-08-01 22:34:33 -04:00
Bailey Dixon 8e4fffab6d fix(android): observe settings pet scroll state 2026-08-01 22:23:32 -04:00
Bailey Dixon 1112a622a7 fix(android): recover pet roaming after scroll 2026-08-01 22:20:03 -04:00
Bailey Dixon 285342bf7e fix(android): keep pet clear of chat scroll control 2026-08-01 22:09:04 -04:00
Bailey Dixon 401acdda8e fix(android): keep floating pet interactive during input 2026-08-01 21:58:22 -04:00
Bailey Dixon a0299d62ca feat(android): preserve pet roaming after drag 2026-08-01 21:49:45 -04:00
Bailey Dixon 4f37b87a3d feat(android): refine floating pet experience 2026-08-01 21:36:34 -04:00
Bailey Dixon 7d3ad1c19f fix(android): make pet visit chat surfaces 2026-08-01 20:55:41 -04:00
Bailey Dixon 54e069888d Merge branch 'docs/pet-experience' into dev 2026-08-01 20:35:28 -04:00
Bailey Dixon 7d3ebfb842 fix(android): localize pet capability previews 2026-08-01 20:35:23 -04:00
Bailey Dixon 7d9552bf1e docs(android): document interactive pet behavior 2026-08-01 20:29:55 -04:00
Bailey Dixon 73c0b6c99d fix(android): label mirrored pet travel accurately 2026-08-01 20:26:09 -04:00
Bailey Dixon 9b68577c47 feat(android): apply pet temperament pacing 2026-08-01 20:23:16 -04:00
Bailey Dixon ec3ff1da02 Merge branch 'feature/android-pet-route-surfaces' into dev 2026-08-01 20:15:30 -04:00
Bailey Dixon 0d78077393 Merge branch 'feature/petdex-capability-preview' into dev 2026-08-01 20:15:29 -04:00
Bailey Dixon eaf345b9c7 Merge branch 'feature/android-pet-temperament' into dev 2026-08-01 20:15:28 -04:00
Bailey Dixon 2d4afd035c Merge branch 'feature/android-pet-core-motion' into dev 2026-08-01 20:15:28 -04:00
Bailey Dixon b5f3eba340 feat(android): make pet roaming intentional 2026-08-01 20:12:29 -04:00
Bailey Dixon e6c48d5fa9 feat(android): preview pet animation capabilities 2026-08-01 20:09:58 -04:00
Bailey Dixon eed739c3a3 feat(android): add pet temperament preferences 2026-08-01 20:09:23 -04:00
Bailey Dixon 91ddebde79 feat(android): add pet roaming to app status chrome 2026-08-01 20:06:19 -04:00
Bailey Dixon a1d99f390f fix(android): keep pet clear of chat content 2026-08-01 19:55:16 -04:00
Bailey Dixon af284952e7 Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 19:49:27 -04:00
Bailey Dixon 4b872f3f54 Merge branch 'fix/android-route-resilience' into feature/android-pet-roaming 2026-08-01 19:46:57 -04:00
Bailey Dixon b725f2b295 feat(android): prefer pet bubble perches 2026-08-01 19:45:55 -04:00
Bailey Dixon 0359fb46ff fix(android): stabilize relay route failover 2026-08-01 19:44:45 -04:00
Bailey Dixon 0e8ab74685 feat(android): enrich pet overlay roaming 2026-08-01 19:38:30 -04:00
Bailey Dixon f49b8d8161 Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 13:57:15 -04:00
Bailey Dixon 8361a059e8 fix(android): avoid stale pet visit targets 2026-08-01 13:46:12 -04:00
Bailey Dixon f6c222ced5 feat(android): animate post-response pet visits 2026-08-01 13:41:27 -04:00
Bailey Dixon 05cda21119 feat(android): schedule post-response pet visits 2026-08-01 13:36:26 -04:00
Bailey Dixon 310893a49a feat(android): add bubble visit routing 2026-08-01 13:36:02 -04:00
Bailey Dixon 09c48efecf fix(android): prioritize pet locomotion states 2026-08-01 13:31:55 -04:00
Bailey Dixon 748c3b1c07 feat(android): add assistant pet visit targets 2026-08-01 13:31:23 -04:00
Bailey Dixon 259d64fe30 fix(android): hold pet drop until state syncs 2026-08-01 13:26:25 -04:00
Bailey Dixon e2044a15ea Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 11:38:13 -04:00
Bailey Dixon 33cc622643 fix(android): preserve manual pet placement 2026-08-01 11:25:01 -04:00
Bailey Dixon 2c388a4c73 fix(android): add obstacle-aware pet hops 2026-08-01 11:21:46 -04:00
Bailey Dixon a4323a6b04 fix(android): smooth pet movement states 2026-08-01 11:15:53 -04:00
Bailey Dixon af6680090f fix(android): refresh pet surface activity 2026-08-01 11:13:56 -04:00
Bailey Dixon 54362c7d31 Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 10:36:02 -04:00
Bailey Dixon 40837013c7 feat(android): add element-aware pet roaming 2026-08-01 10:35:49 -04:00
Bailey Dixon 70edc5e73f Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 09:48:55 -04:00
Bailey Dixon 55e8486f94 fix(android): animate pet roam locomotion 2026-08-01 09:48:47 -04:00
Bailey Dixon ab4519d4cf Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 09:35:50 -04:00
Bailey Dixon d5cfa2bf5c fix(android): match desktop pet overlay behavior 2026-08-01 09:35:34 -04:00
Bailey Dixon c5376b2c25 Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 09:14:19 -04:00
Bailey Dixon ee33666e1f fix(android): dock pet at chat end edge by default 2026-08-01 09:14:14 -04:00
Bailey Dixon c357f201c6 Merge branch 'feature/android-pet-roaming' into dev 2026-08-01 08:53:18 -04:00
Bailey Dixon 17a439cfaa feat(android): add roaming Petdex companions 2026-08-01 08:53:04 -04:00
Bailey Dixon 6b888e91d3 Merge Petdex preview cache follow-up into dev 2026-07-31 23:47:09 -04:00
Bailey Dixon 0793f9213c perf(android): prioritize cached pet previews 2026-07-31 23:47:04 -04:00
Bailey Dixon 146652e475 Merge Petdex preview retry follow-up into dev 2026-07-31 23:45:33 -04:00
Bailey Dixon 9bdaf3a02f fix(android): retry Petdex preview loading 2026-07-31 23:45:27 -04:00
Bailey Dixon e08d0e13a4 Merge branch 'feature/petdex-previews' into dev 2026-07-31 23:43:43 -04:00
Bailey Dixon 3c10c67075 feat(android): add Petdex gallery previews 2026-07-31 23:43:38 -04:00
Bailey Dixon e8e51d9ee4 Merge branch 'feature/android-floating-pet' into dev
# Conflicts:
#	CHANGELOG.md
#	app/src/main/kotlin/com/hermesandroid/relay/ui/components/MessageBubble.kt
2026-07-31 22:59:35 -04:00
Bailey Dixon dd5a80d03c fix(android): recover stalled voice output 2026-07-31 22:58:34 -04:00
Bailey Dixon 06a2f35144 feat(android): add floating pets and Petdex 2026-07-31 22:57:30 -04:00
Bailey Dixon 3d78e79c5d feat(android): refine voice mode and overlay 2026-07-31 22:35:18 -04:00
Bailey Dixon 50f745d9da Merge branch 'fix/android-standard-voice-oom' into dev
# Conflicts:
#	CHANGELOG.md
#	app/src/test/kotlin/com/hermesandroid/relay/network/upstream/StandardHermesVoiceClientTest.kt
2026-07-31 21:46:04 -04:00
Bailey Dixon 33e8a11615 fix(android): stream standard voice transcription uploads 2026-07-31 21:43:12 -04:00
Bailey Dixon 21bbad5f3b Merge branch 'fix/android-r8-resource-shrinking' into dev 2026-07-31 21:04:30 -04:00
Bailey Dixon 6524157549 fix(android): enable release resource shrinking 2026-07-31 21:04:17 -04:00
Bailey Dixon e8192b09dd docs(android): clarify voice stop dependency 2026-07-31 19:45:31 -04:00
Bailey Dixon ef3916a143 feat(android): align voice interruption with upstream 2026-07-31 19:30:53 -04:00
Bailey Dixon f2b92b2755 Merge pull request #274 from Codename-11/fix/android-1.5.3-voice-transcript-keys
release(android): Android 1.5.3 duplicate-key hotfix
2026-07-31 19:03:40 -04:00
Bailey Dixon 8651656899 release(android): android-v1.5.3 2026-07-31 18:50:21 -04:00
Bailey Dixon b458d83fcc fix(android): stabilize voice transcript keys 2026-07-31 18:27:59 -04:00
Bailey Dixon f4ae8d21ca fix(android): preserve passport shell in identity editor 2026-07-31 17:26:56 -04:00
Bailey Dixon b2f8070a1b feat(android): refine agent passport controls 2026-07-31 17:05:49 -04:00
Bailey Dixon 27f1393ea7 Merge branch 'feature/android-assistant-overlay' into dev 2026-07-31 15:44:26 -04:00
Bailey Dixon d8f8082639 feat(android): unify assistant voice surfaces 2026-07-31 15:44:20 -04:00
Bailey Dixon 5df42c1fda Merge branch 'feature/android-agent-passport-v2' into dev 2026-07-31 15:15:07 -04:00
Bailey Dixon 778c15de7f feat(android): deepen agent passport details 2026-07-31 15:14:58 -04:00
Bailey Dixon a606eb7c40 Merge branch 'fix/android-assistant-picker' into dev 2026-07-30 19:53:17 -04:00
Bailey Dixon a4df726bae fix(android): support voice-task assistant launch 2026-07-30 19:53:12 -04:00
Bailey Dixon f87de0f96a Merge branch 'fix/android-assistant-picker' into dev 2026-07-30 19:46:49 -04:00
Bailey Dixon 2471c3dd55 fix(android): activate selected assistant service 2026-07-30 19:46:42 -04:00
Bailey Dixon c53b0c6aa3 Merge branch 'fix/android-assistant-picker' into dev 2026-07-30 19:38:49 -04:00
Bailey Dixon 8ade8debf0 fix(android): expose assistant picker entry point 2026-07-30 19:38:42 -04:00
Bailey Dixon e84eeb8e4f Merge branch 'fix/android-wake-strictness-default' into dev 2026-07-30 19:23:52 -04:00
Bailey Dixon d28f0d5de7 fix(android): tune wake strictness defaults 2026-07-30 19:23:47 -04:00
Bailey Dixon c680f6f896 Merge branch 'feature/android-full-assistant-mode' into dev 2026-07-30 19:15:34 -04:00
Bailey Dixon 7df350365c feat(android): add opt-in digital assistant mode 2026-07-30 19:02:06 -04:00
Bailey Dixon 0795565a4d Merge branch 'feature/android-voice-wake-parity' into dev 2026-07-30 18:39:58 -04:00
Bailey Dixon fe4ef2441c fix(android): resume enabled wake listener visibly 2026-07-30 18:39:53 -04:00
Bailey Dixon 042f5c0927 Merge branch 'fix/android-voice-transcript-keys' into dev
# Conflicts:
#	CHANGELOG.md
#	DEVLOG.md
2026-07-30 18:28:01 -04:00
Bailey Dixon 2998773e70 fix(android): stabilize voice transcript keys 2026-07-30 18:27:31 -04:00
Bailey Dixon 91ab611b54 Merge branch 'feature/android-voice-wake-parity' into dev 2026-07-30 18:23:20 -04:00
Bailey Dixon c76d0bf33e fix(android): stabilize local wake activation 2026-07-30 18:23:05 -04:00
Bailey Dixon ff965305d2 Merge branch 'feature/android-voice-wake-parity' into dev 2026-07-29 21:20:31 -04:00
Bailey Dixon a7d76d56e5 feat(android): absorb upstream voice and wake parity 2026-07-29 20:24:16 -04:00
Bailey Dixon 6e1e3d8b38 Merge branch 'feature/open-ledger-batches' into dev 2026-07-28 21:49:40 -04:00
Bailey Dixon 04944ffe8d feat(android): consume upstream profile and gateway contracts 2026-07-28 21:47:27 -04:00
Bailey Dixon 34cf109804 Merge origin/dev into dev 2026-07-28 21:05:11 -04:00
Bailey Dixon e459f24a1a Merge branch 'fix/android-agent-passport-drawer' into dev 2026-07-28 21:04:53 -04:00
Bailey Dixon 4346e33fd4 fix(android): repair agent passport drawer flow 2026-07-28 21:04:47 -04:00
Bailey Dixon ca0c5b2a54 Merge pull request #266 from Codename-11/fix/android-https-gateway-route
fix(android): preserve secure gateway routes
2026-07-28 21:04:31 -04:00
Bailey Dixon 77e34c2c02 fix(android): preserve secure gateway routes 2026-07-28 19:35:14 -04:00
Bailey Dixon f4ee409106 docs(devlog): record Android 1.5.2 release 2026-07-28 18:20:37 -04:00
Bailey Dixon aa26f7c9b6 Merge pull request #265 from Codename-11/dev
release(android): android-v1.5.2
2026-07-28 17:58:37 -04:00
Bailey Dixon bfb608bea6 release(android): android-v1.5.2 2026-07-28 17:30:59 -04:00
Bailey Dixon 95a95fe7d2 Merge pull request #264 from Codename-11/fix/android-nous-native-auth
fix(android): support Nous system-browser sign-in
2026-07-28 17:28:56 -04:00
Bailey Dixon 1bdf2ae71b fix(android): support Nous system-browser sign-in 2026-07-28 17:15:26 -04:00
Bailey Dixon f9e7a2f320 Merge pull request #263 from Codename-11/fix/android-duplicate-compose-keys
fix(android): coalesce replayed chat message ids
2026-07-27 11:38:06 -04:00
Bailey Dixon 988fac8522 Merge pull request #262 from Codename-11/fix/android-oidc-manage-callback
fix(android): keep dashboard OIDC on cookie flow
2026-07-27 11:37:43 -04:00
Bailey Dixon d4832a6a38 fix(android): coalesce replayed chat message ids 2026-07-27 09:30:39 -04:00
Bailey Dixon f9c8736e5b fix(android): keep dashboard OIDC on cookie flow 2026-07-27 08:57:17 -04:00
dependabot[bot] a815dd33fa build(deps): bump com.android.library from 9.3.0 to 9.3.1 (#261)
Bumps com.android.library from 9.3.0 to 9.3.1.

---
updated-dependencies:
- dependency-name: com.android.library
  dependency-version: 9.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-07-27 11:56:57 +00:00
dependabot[bot] cc9c75a636 build(deps): bump androidx.browser:browser from 1.9.0 to 1.10.0 (#260)
Bumps androidx.browser:browser from 1.9.0 to 1.10.0.

---
updated-dependencies:
- dependency-name: androidx.browser:browser
  dependency-version: 1.10.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-07-27 11:55:03 +00:00
dependabot[bot] 43179e03c0 build(deps): bump com.android.application from 9.3.0 to 9.3.1 (#259)
Bumps com.android.application from 9.3.0 to 9.3.1.

---
updated-dependencies:
- dependency-name: com.android.application
  dependency-version: 9.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-07-27 11:55:00 +00:00
Bailey Dixon 325b8e5670 Merge pull request #257 from Codename-11/dev
release(android): promote android-v1.5.1
2026-07-26 16:11:49 -04:00
Bailey Dixon 693ac4ed64 Merge pull request #256 from Codename-11/release/android-1.5.1-patch
release(android): android-v1.5.1
2026-07-26 15:44:19 -04:00
Bailey Dixon f94d663ac4 release(android): android-v1.5.1 2026-07-26 15:35:21 -04:00
Bailey Dixon d1a21bd42e fix(android): use Compose resources for sign-in copy 2026-07-26 15:35:20 -04:00
Bailey Dixon 7ee2d73010 fix(android): preserve formatted chat completion position 2026-07-26 15:10:48 -04:00
Bailey Dixon 682bde84fe fix(android): merge API 36 target 2026-07-26 09:38:22 -04:00
Bailey Dixon 16bdbe5f44 fix(android): target API 36 2026-07-26 09:38:00 -04:00
Bailey Dixon f43fba9fed feat(android): merge chat readability and final-only voice 2026-07-26 09:13:57 -04:00
Bailey Dixon d1745413fd fix(android): release realtime background foreground turns 2026-07-26 08:57:00 -04:00
Bailey Dixon 1b7a8025c3 fix(android): restore standard voice narration 2026-07-26 08:56:42 -04:00
Bailey Dixon d92a87483e feat(android): enhance voice conversation experience 2026-07-26 08:56:23 -04:00
Bailey Dixon e9da59cf40 Merge pull request #254 from Codename-11/dev
fix(android): repair immutable release dispatch
2026-07-25 18:30:53 -04:00
Bailey Dixon 0bea626ed8 Merge pull request #253 from Codename-11/fix/android-release-dispatch
fix(android): repair immutable release dispatch
2026-07-25 18:30:27 -04:00
Bailey Dixon c9a03c9ed7 Merge pull request #252 from Codename-11/dev
release(android): android-v1.5.0
2026-07-25 18:26:34 -04:00
Bailey Dixon 68f8b58a0b Merge pull request #232 from Codename-11/dev
release(android): android-v1.4.9
2026-07-19 21:27:28 -04:00
250 changed files with 37300 additions and 2573 deletions
+63
View File
@@ -10,6 +10,69 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
- **Windows-trusted certificates work in the desktop CLI.** The packaged Windows binary and newer Node runtimes add the Windows certificate store without dropping bundled or operator-supplied roots, while TLS verification and Relay certificate pinning remain enforced.
## [Android 1.6.0] - 2026-08-02
### Added
- **Hermes can be selected as Android’s default Digital Assistant.** The opt-in system role supports background and locked-screen invocation, while the separate experimental “Hey Hermes” listener keeps pre-activation audio on the phone and exposes an ongoing Stop control.
- **Installed Hermes plugins can contribute native Android pages.** Android renders a bounded declarative schema instead of plugin code, keeps write access off until the user grants it, and supports approval-gated agent-created previews through Relay 1.5.0.
- **Pets can stay with you across the Android app without replacing the agent.** Petdex and imported companions live in an app-level overlay, can be held and dragged, and optionally roam across live-measured chat and settings surfaces without reserving message space. (#267)
- **Petdex browsing and one-tap installation are built into Appearance.** Search results use lightweight previews, full atlases download only after Install, creator attribution remains visible, and installed pets stay available offline. (#267)
- **Android can be used in Russian.** Both product flavors include an AI-assisted Russian catalog, language picker support, localized plurals, and refreshed translations for the 1.6 feature set.
### Changed
- **Assistant and floating Voice surfaces use compact, expandable controls.** Opening full Voice continues the same turn and microphone owner instead of restarting the session.
- **Voice interruption covers generation and playback.** Barge-in follows upstream RMS calibration and timing, exact stop phrases can end an active voice chat, and interrupted spoken context remains private to the next Standard turn.
- **Profile identity, the Sphere, and pets are separate appearance choices.** Agent avatars identify messages, background visualization controls ambient art, and Floating pet controls the companion independently. (#267)
- **The Agent Passport exposes more profile state and safer controls.** Profile configuration, skills, routing, reasoning, and scoped API access remain visibly distinct from the active session identity.
### Fixed
- **Voice output recovers when a streaming renderer produces no audio.** Android falls back to basic synthesis after a bounded first-audio timeout, and long Standard Voice uploads no longer retain duplicate encoded audio buffers.
- **Relay route failover avoids competing reconnect loops.** Route changes settle through one generation-aware reconnect owner instead of rapidly switching between LAN and remote candidates.
- **Live chat rows keep stable UI identity while upstream state reconciles.** Streamed messages and process rows no longer collide or restart merely because a server identity arrives later.
- **Floating pets recover from invalid or scrolling terrain.** Roaming uses measured bubble edges, avoids the jump-to-latest control and text overlap, resumes after drag or scrolling, and preserves locomotion, held, drop, and fallback animation states.
- **Hermes appears and activates in OEM Android assistant pickers.** Required Assist, Voice, recognition-service, and single-microphone lifecycle metadata now agree.
- **Experimental wake detection handles completed sherpa results and empty speech cleanly.** Tests use the real local microphone/model path, and no-speech activation returns to ready state instead of surfacing a fatal server error.
## [Android 1.5.3] - 2026-07-31
### Fixed
- **Voice transcripts retain stable rows after chat-history reconciliation.** Focus mode uses the same stable Compose identity as the main conversation, preventing duplicate-key crashes when live rows adopt persisted server IDs.
## [1.5.0] - 2026-08-02
### Added
- **Realtime Agent sessions can speak only settled answers.** Clients may enable an optional per-session `final_answer_only` policy that suppresses routine acknowledgements, progress narration, and intermediate commentary while preserving spoken approvals, confirmation questions, blocking failures, and the final Hermes answer.
- **Agents can draft native Android plugin pages through Relay.** New tools store bounded declarative JSON pages under the authenticated Relay plugin namespace, while Android retains control of enablement, publication, write grants, and persistent removal. Generated pages cannot include executable code, arbitrary network calls, Android intents, or backend action requests.
## [Android 1.5.2] - 2026-07-28
### Fixed
- **Dashboard sign-in completes across supported providers and network routes.** Self-hosted OIDC stays on the dashboard cookie flow, while Nous Portal opens in the system browser and completes standards-compatible PKCE through HTTPS, private-LAN, or Tailscale dashboard routes.
- **Replayed chat updates no longer destabilize the conversation list.** Duplicate upstream message identifiers are coalesced before Compose renders them.
## [Android 1.5.1] - 2026-07-26
### Added
- **Voice supports focused and conversational layouts.** Focus keeps spoken turns, Markdown, tools, media, and actions in a compact voice surface, while Conversation opens the full Chat renderer without leaving the active voice session.
- **Voice can speak only settled answers.** A global Voice setting keeps tool progress, service updates, and intermediate commentary visual while supported voice paths wait to speak the final Hermes answer.
### Changed
- **Chat answers are easier to read in every theme.** Primary assistant text now uses the theme's full-contrast foreground, and chat prose uses a 15sp size with 21sp line height.
- **Google Play builds target Android 16.** The app now targets API level 36 while retaining its existing minimum-device support.
### Fixed
- **Completed streamed answers render their formatting without losing the reading position.** Markdown headings, lists, emphasis, and code blocks replace the live text renderer only after completion, then the measured trailing edge remains anchored at the bottom.
- **Standard Voice speaks completed assistant replies again.** Session and message fences no longer suppress a valid final answer during the handoff from generation to narration.
- **Realtime background work no longer blocks the active voice controls.** A promoted task releases the foreground spinner and microphone while its progress, tools, cancellation, and final result remain available in the owning chat.
## [Server 1.4.3] - 2026-07-22
### Added
+219
View File
@@ -1,5 +1,211 @@
# Hermes-Relay — Dev Log
## 2026-08-02 — Android Russian localization
Android now ships complete Russian catalogs for the main and sideload builds.
The in-app language picker, Android locale configuration, chat and voice labels,
tool and status presentation, diagnostics, onboarding, and plural resources are
registered against the canonical English catalog. Existing non-English catalogs
were refreshed to retain exact resource and format-argument parity.
The integration preserves PR #276 as the source contribution while excluding
unrelated recovery, routing, and test-stability changes from the localization
scope. The localization registry records Android coverage only; Russian public
documentation and marketing pages continue to use the canonical English
fallback until those surfaces are translated separately.
## 2026-07-31 — Upstream-compatible voice interruption semantics
Android full-turn barge-in now follows upstream Hermes' RMS behavior: roughly
450 ms of quiet-room calibration, a 90th-percentile floor, 3× default
multiplier, separate generation/playback minimums, a bounded ceiling, 500 ms
playback grace, and an 80%-majority decision window. Calibration remains frozen
against speaker output and cannot itself trigger. Renderer-driven phase tracking
returns to generation thresholds in quiet output gaps and rearms playback grace
only after a gap of at least one second. Opt-in Logcat diagnostics expose the
inputs used for device tuning. Barge-in is enabled by default while retaining its master,
Silero sensitivity, RMS multiplier, playback grace, and resume controls.
Voice stop phrases now use an editable exact-match list that defaults to
`stop`; clearing the list disables the behavior. A match ends the active voice
chat in generation or playback, but the same word outside voice chat and longer
requests continue through normal Hermes input. Continuous-mode pause/resume and
explicit background-task cancellation retain their narrower state gates.
Interrupting spoken playback arms the upstream one-shot interruption note for
the next Standard model-bound message. The latch expires after 120 seconds and
travels only in API-local voice interface context, never in visible or
persisted user text. Generation and pre-audio synthesis interruption do not mark
an unspoken reply, Realtime keeps its provider-session context, and silencing remains independent
from cancellation of promoted background work.
## 2026-07-30 — Expandable Android assistant surface
Android Digital Assistant sessions now open as a compact bottom bar over the
current app, expand in place for transcript and response detail, and collapse
without changing the active turn. Open full voice disables only the
system-owned session UI and reveals the existing app Voice surface, preserving
the same session, response stream, and microphone owner.
Connection, chat, and voice state machines now have one main-process,
application-lifetime owner. Assistant activation can initialize and run a cold
voice turn without constructing or foregrounding `MainActivity`; opening full
Voice binds the Activity to those same ViewModels and audio resources.
The optional `SYSTEM_ALERT_WINDOW` Voice surface now follows the same
wide-bar-to-expanded-sheet progression while retaining its minimized bubble.
It remains a separately user-invoked control for turns that began in the app;
the Assistant-role session does not require display-over-other-apps permission.
The assistant window is transparent outside the bar or sheet, leaves the
underlying app unresized, and restricts touch interception to the measured
surface. Back collapses an expanded surface first; Back from compact, Stop, and
ordinary dismissal remain terminal. A hidden full-Voice handoff instead follows
the app-owned turn through its final Closed state.
Package-scoped lifecycle reconciliation also clears assistant state if Android
reclaims the separately processed UI while full Voice remains active.
Assistant activation is ID-aware and single-flight. Duplicate delivery cannot
re-arm capture, Retry replaces a pending readiness attempt, and Stop invalidates
the attempt before chat, Voice, or microphone mutations. Scoped voice settings
must hydrate from DataStore before route readiness, and process extraction
preserves connection-catalog isolation plus the existing gateway route-flip
settle window.
Physical-device validation on a Samsung SM-S938U confirmed cold invocation over
a non-Hermes foreground app, compact and expanded presentation without
foregrounding `MainActivity`, and one main-process microphone owner. Locked
invocation reached a shown system assistant session without runtime or recorder
errors; Samsung's secure lock screen prevented screenshot-based visual review.
## 2026-07-30 — Foreground wake-word diagnostics and recovery
The Android-local sherpa listener now treats each non-empty keyword result as a
completed KWS event, resets the stream immediately, and maps the stored
confirmation setting to sherpa's native trailing-blank confirmation instead of
requiring an already-completed result to recur across application frames. Voice
settings can arm a ten-second test against the same foreground service,
microphone owner, installed model, and current tuning; it displays live input
level and reports detection without opening voice or transmitting audio.
Expected empty-transcript responses after activation now record a no-speech
diagnostic and return voice to its ready state with a retry hint rather than
surfacing the provider's HTTP error. Other transcription failures retain the
existing error path. Foreground-service behavior is documented explicitly:
background detections remain pending behind the notification until Hermes is
visible; Android default-assistant integration is a separate mode. Opening the
visible Voice settings screen also reconciles an enabled listener after package
replacement or process death without adding boot/background auto-start.
Focused wake preferences/core and no-speech classification tests pass.
Sideload lint, sideload debug packaging, and Google Play debug Kotlin
compilation pass. This batch adds no model, native library, ABI, permission, or
network dependency; the existing approximately 6 MB downloaded model and
packaged sherpa ABI footprint are unchanged.
## 2026-07-30 — Voice transcript identity alignment
Android voice Focus mode now keys transcript rows with the same stable UI
identity as the main Chat list. A live row may adopt its persisted server
message ID during history reconciliation while retaining its original Compose
identity; using the mutable domain ID in the voice overlay could otherwise
collide during that transition and close the app.
Focused JVM coverage recreates two visible rows with a shared reconciled server
ID and verifies distinct stable transcript keys. Sideload production and
Android-test Kotlin compilation pass. The existing full-overlay instrumentation
fixture remains blocked by its continuously animating surface never reaching
Compose idleness.
## 2026-07-30 — Opt-in Android Digital Assistant mode
Android now declares an explicit `VoiceInteractionService` and separately
processed `VoiceInteractionSessionService`. Only Android's user-confirmed
Assistant role activates the integration. Optional background “Hey Hermes”
detection reuses the local sherpa model and tuning, releases its recorder before
the system session opens the existing voice flow, and resumes after session
exit. Package-scoped lifecycle messages reconcile prompt/listen state,
transcript/response presentation, cancellation, errors, and process recreation.
The Digital Assistant listener and the existing experimental microphone
foreground service are separate, mutually exclusive opt-ins. Both retain local
pre-activation privacy and the shared one-microphone contract. Standard voice
continues through the upstream Dashboard audio surface. Voice settings include
role status, setup, removal, runtime status, and the limitation that third-party
assistants do not receive Google's low-power hotword hardware.
## 2026-07-29 — Full-turn voice interruption and local wake-word preview
Android barge-in now owns one microphone/VAD listener from response generation
through playback drain for both Standard and Realtime voice. Quiet-room RMS
calibration freezes before output begins, playback receives a grace interval,
and model-confirmed majority filtering separates actual interruption from raw
ducking hints. Turn epochs, stream cancellation, late-delta suppression, and
an awaited microphone handoff keep an interrupted response from speaking again
or racing the replacement recording. Exact stop/pause intent is phase-aware,
while explicit background-task cancellation remains separate from silencing.
An opt-in Android-local “Hey Hermes” preview uses sherpa-onnx in a user-started
microphone foreground service. Its approximately 6 MB English model is
downloaded and hash-verified on first enable rather than bundled. The service
keeps pre-activation audio local, exposes an ongoing Stop notification, pauses
for active voice, and shares a process-wide single-microphone ownership
contract with voice recording, barge-in, and realtime diagnostics. The stored
configuration includes strictness, confirmation frames, new-session behavior,
and a deliberately inactive future profile-routing shape.
Focused JVM coverage exercises calibration, grace, listener teardown,
Thinking-to-Speaking ownership, generation/playback interruption, command
gating, wake preferences, activation, and microphone exclusion. Android
compilation for both distribution flavors, sideload lint, and sideload APK
packaging pass with all four supported ABIs. On-device acoustic, foreground
service, and lifecycle checks remain the corresponding validation gates.
## 2026-07-28 — Android 1.5.2 production release
Android 1.5.2 shipped from the approved `dev` to `main` release tree as
versionCode 35. The release adds provider-aware Dashboard sign-in: Nous uses
the advertised native PKCE system-browser flow, while compatible self-hosted
providers retain cookie-backed full-page Dashboard authentication. Callback
origin discovery remains server-driven, private-network HTTP compatibility is
preserved, and arbitrary public HTTP redirects remain rejected.
The private Play preflight validated the exact application tree before release
PR #265 merged. The immutable `android-v1.5.2` tag resolves to the resulting
`main` tip, the production workflow promoted versionCode 35 to the completed
Google Play production track, and the public GitHub release contains the
signed AAB, sideload APK, and SHA-256 manifest. The published sideload APK
checksum was independently verified; replacing the debug-signed phone build
with the release-signed artifact requires an uninstall because Android
correctly rejects cross-signature in-place updates.
## 2026-07-27 — Android replayed-message identity reconciliation
Android history reconciliation now collapses reconnect/rejoin replays of the
same persisted message ID before publishing the transcript to Compose. The
latest repeated snapshot replaces the value at the message's first transcript
position, preserving stable ordering, distinct messages, and the LazyColumn
identity contract without index- or random-key fallbacks.
Focused coverage reproduces the duplicate UUID condition and verifies that the
authoritative final content wins while every rendered message keeps a unique
stable UI key.
## 2026-07-26 — Android 1.5.1 patch reconciliation
Android 1.5.1 reconciles the post-1.5.0 voice and chat fixes into versionCode
34. Voice now offers compact Focus and full Conversation presentation,
Standard narration preserves valid completed replies, and promoted Realtime
tasks release foreground voice controls while retaining progress and results.
Completed streamed answers promote from the stable live text node to full
Markdown only after completion. The measured Markdown row is then positioned
by its trailing edge until deferred code and attachment measurement settles,
preventing the LazyColumn from restoring the start of a tall response.
The release also targets Android API level 36. Release notes, in-app What's
New assets, localized Play notes, and the Play listing reference were updated
for Android 1.5.1.
## 2026-07-25 — Immutable Android release dispatch repair
Android approval now dispatches the current release workflow definition from
@@ -6698,3 +6904,16 @@ After: Phone (HTTP/SSE) → API Server (:8642) [chat — direct]
- Deploy docs site (GitHub Pages or similar)
- Phase 2: Terminal channel (xterm.js in WebView, tmux integration)
- Phase 3: Bridge channel migration
# 2026-07-30 — Wake-word strictness tuning
- Lowered the unset Android wake-word strictness default from `0.6` to `0.3` after physical-device testing showed reliable activation only at the lower slider positions.
- Added the live numeric strictness value to Voice settings so tuning is observable.
- Preserved saved user values and the existing three-frame confirmation default; this adjustment does not migrate working installations or broaden the detector acceptance window beyond the selected threshold.
# 2026-07-30 — OEM assistant-picker compatibility
- Added the standard `android.intent.action.ASSIST` activity filter because some OEM assistant pickers enumerate Assist activities even when a valid `VoiceInteractionService` is present.
- Routed activity-based Assist invocations through the existing system-assistant activation protocol so both Android entry points share microphone ownership and session lifecycle behavior.
- Added the required `recognitionService` metadata and a bounded recognition component after device validation showed Samsung could grant the package role while leaving the active voice-interaction service empty. The component does not open the microphone; Hermes assistant sessions continue to use the established transcription pipeline.
- Declared `CATEGORY_VOICE` on the Assist activity and retained `ACTION_ASSIST` on the explicit activation intent. `VoiceInteractionSession.startVoiceActivity()` adds the voice category, and Android rejects the launch as `START_NOT_VOICE_COMPATIBLE` unless the target filter matches both.
+7 -12
View File
@@ -1,22 +1,17 @@
# Hermes-Relay-Plugin v__VERSION__
# Hermes-Relay-Server v__VERSION__
**Release Date:** July 22, 2026
**Release Date:** August 2, 2026
This patch hardens Relay authorization, adds upstream-aware diagnostics, and keeps plugin bootstrap work off the Gateway event loop.
This release adds Realtime Agent final-answer-only speech and Relay-hosted declarative plugin pages for Android.
It can accompany Hermes-Relay-Android v1.5.0 for optional Relay diagnostics and power features. Standard chat and Vanilla Hermes voice remain upstream-owned and do not require this plugin.
Android clients can keep voice progress visual until the settled answer and review agent-created native plugin pages before keeping them. Standard chat and Vanilla Hermes voice remain upstream-owned and do not require this plugin.
## What's changed
### Added
- **Upstream-aware Gateway diagnostics.** Doctor and `/relay/info` expose optional health, configuration-route, and capability signals so clients can explain compatibility gaps without treating an older upstream install as a broken Relay.
### Fixed
- **Privileged Relay paths enforce host authorization and active grants.** Pairing, Android bridge, terminal, session policy, remote profile configuration, and voice provider origins retain their intended trust boundaries.
- **Plugin bootstrap remains responsive.** Database initialization and compatibility inspection run outside the Gateway event loop while preserving compatibility with older upstream bootstrap contracts.
- **Windows Gateway detection is non-signalling.** Starting Relay and periodic profile rescans no longer risk terminating an existing Gateway process.
- **Realtime Agent final-answer-only speech.** Session creation accepts an optional `final_answer_only` flag. When enabled, the provider skips routine acknowledgements, spoken progress, service updates, and intermediate commentary, then speaks the settled Hermes answer. Approval and confirmation prompts, along with blocking failures, remain audible so required user action is not hidden.
- **Agent-created declarative Android plugin pages.** New Relay tools create bounded JSON-only drafts and expose them through authenticated plugin routes. Android owns enablement, write grants, exact-revision approval, publication, and persistent removal. Generated pages reject executable code, arbitrary network requests, Android intents, traversal, symlink entries, oversized documents, and backend action requests.
## Install / update
@@ -24,7 +19,7 @@ It can accompany Hermes-Relay-Android v1.5.0 for optional Relay diagnostics and
hermes plugins install Codename-11/hermes-relay/plugin --enable
# Classic install / update on a systemd host:
curl -fsSL https://raw.githubusercontent.com/Codename-11/hermes-relay/main/install.sh | bash
curl -fsSL https://raw.githubusercontent.com/Codename-11/hermes-relay/server-v__VERSION__/install.sh | bash
# or, if already installed:
hermes-relay-update
+9 -6
View File
@@ -34,10 +34,10 @@
Hermes-Relay puts your [Hermes agent](https://github.com/NousResearch/hermes-agent) on the devices you actually carry. The brain stays on your own machine — Hermes-Relay is how you reach it.
- **📱 Android app** — streaming chat, hands-free voice, and the full Hermes dashboard (models, keys, skills, profiles), rebuilt native. On sideload builds, the agent can read your screen and act on it.
- **📱 Android app** — streaming chat, hands-free voice, native plugin pages, and the full Hermes dashboard (models, keys, skills, profiles), rebuilt native. Add a floating Petdex companion or optionally make Hermes your Android assistant; sideload builds can also let the agent read and act on your screen.
- **⌨️ Hermes-Relay CLI** *(alpha)* — a single binary that gives the agent **hands on any machine you pair**: files, terminal, search, screenshots — consent-gated.
A vanilla [hermes-agent](https://github.com/NousResearch/hermes-agent) install is enough — chat, management, and voice need **no plugin**. Add the optional relay only when you want terminal, phone control, or the CLI's tools. **Pair once from either surface; both work.**
A vanilla [hermes-agent](https://github.com/NousResearch/hermes-agent) install is enough — chat, management, voice, Petdex, and ordinary installed-plugin pages need **no Relay plugin**. Add the optional Relay only when you want terminal, phone control, agent-created page drafts, or the CLI's tools. **Pair once from either surface; both work.**
<p align="center">
<img src="docs/diagrams/architecture-homepage.png" alt="How Hermes-Relay connects — Vanilla Hermes (Chat, Manage, Voice) runs with no plugin; the optional Relay plugin adds Terminal, Bridge, relay voice and desktop tools to the app and CLI; Device Control needs the sideload build." width="900">
@@ -99,7 +99,7 @@ the whole Vanilla Hermes setup.
### 4 · Optional: install Relay for power tools
Install the Relay plugin on the server only when you want Terminal, Bridge phone control, relay sessions, media routes, or the realtime voice engine:
Install the Relay plugin on the server only when you want Terminal, Bridge phone control, relay sessions, media routes, the realtime voice engine, or approval-gated agent-created plugin-page drafts:
```bash
hermes plugins install Codename-11/hermes-relay/plugin --enable
@@ -115,8 +115,11 @@ shell shims, and the full clone/update workflow:
curl -fsSL https://raw.githubusercontent.com/Codename-11/hermes-relay/main/install.sh | bash
```
The plugin-manager install owns the plugin code, dashboard tab, CLI commands,
and agent tools. `hermes relay compat status/install/remove` manages only the
Installed Hermes plugins can expose bounded, host-rendered pages to Android
through the authenticated Dashboard without running plugin code on the phone.
Relay 1.5.0 additionally supports approval-gated agent-created page drafts. The
plugin-manager install owns the plugin code, dashboard tab, CLI commands, and
agent tools. `hermes relay compat status/install/remove` manages only the
optional legacy API compatibility hook when an older Hermes build needs it. Scan
the QR from the phone's Connections screen — or use
`hermes pair --register-code ABCD12` with the manual code from Android
@@ -159,7 +162,7 @@ Full server setup, TLS, and systemd details: [docs/relay-server.md](docs/relay-s
</table>
The Android app ships complete AI-assisted catalogs for **Deutsch**, **Español**,
**日本語**, **Português (Brasil)**, and **简体中文**. Choose a language from
**日本語**, **Português (Brasil)**, **Русский**, and **简体中文**. Choose a language from
**Settings → Appearance → Language**; translation status and fluent review are
tracked independently so community corrections remain easy to contribute.
+5 -3
View File
@@ -18,9 +18,9 @@
## 功能简介
- **Android 应用**:流式聊天、会话历史、文件附件、Hermes 管理、语音模式、多连接和配置文件。
- **Android 应用**:流式聊天、会话历史、文件附件、Hermes 管理、语音模式、原生插件页面、Petdex 悬浮宠物、多连接和配置文件;也可将 Hermes 设为 Android 助手。
- **无需插件的标准路径**:聊天、管理和标准语音可直接连接未修改的上游 Hermes Agent。
- **可选 Relay 插件**:增加终端、手机控制、媒体传输、通知助手、Relay 语音和电脑工具。
- **可选 Relay 插件**:增加终端、手机控制、媒体传输、通知助手、Relay 语音、电脑工具,以及需确认的代理创建插件页面草稿。
- **安全连接**:二维码配对、Android Keystore、证书固定、按通道授权和可配置会话有效期。
- **远程使用**:可配置 Tailscale 或 HTTPS 地址,在家庭局域网和远程路由之间自动切换。
- **两种 Android 发行渠道**:Google Play 版本适合日常使用;sideload 版本包含完整手机控制能力。
@@ -67,7 +67,7 @@ hermes gateway
### 4. 可选:安装 Relay
仅在需要终端、手机控制、媒体路由、Relay 会话、实时语音或电脑工具时安装:
仅在需要终端、手机控制、媒体路由、Relay 会话、实时语音、电脑工具或代理创建插件页面草稿时安装:
```bash
hermes plugins install Codename-11/hermes-relay/plugin --enable
@@ -76,6 +76,8 @@ hermes relay start --no-ssl
hermes pair
```
已安装的 Hermes 插件可通过已认证的 Dashboard 向 Android 提供由应用安全渲染的原生页面,无需在手机上运行插件代码。Relay 1.5.0 另支持需用户确认的代理创建页面草稿。
完整说明请阅读[中文快速开始](https://hermes-relay.dev/docs/zh-CN/guide/quick-start);远程访问、协议和高级配置暂时链接到英文参考文档。
## 中文界面
+24 -18
View File
@@ -1,10 +1,10 @@
# Hermes-Relay-Android v1.5.0
# Hermes-Relay-Android v1.6.0
**Release Date:** July 25, 2026
**Release Date:** August 2, 2026
## Download
> Installing on your phone? Download `hermes-relay-1.5.0-sideload-release.apk` and tap it for the full feature set, or install the conservative build from [Google Play](https://play.google.com/store/apps/details?id=com.axiomlabs.hermesrelay).
> Installing on your phone? Download `hermes-relay-1.6.0-sideload-release.apk` and tap it for the full feature set, or install the conservative build from [Google Play](https://play.google.com/store/apps/details?id=com.axiomlabs.hermesrelay).
The `.aab` file is a Play Console upload bundle and cannot be installed by tapping it on a phone.
@@ -12,28 +12,34 @@ Verify the download against `SHA256SUMS.txt`. See the [sideload guide](https://h
## Summary
This release makes the Hermes Dashboard and Gateway the clear standard connection, keeps active work reachable across the app and Android backgrounding, and gives profiles, voice, attachments, image generation, and approvals a more coherent native interface.
This minor release brings personality and extensibility to Android: floating Petdex companions can explore the interface, Hermes plugins can contribute safe native pages, and Hermes can become the optional Android Digital Assistant. Voice, route failover, live-list identity, and Russian localization are also substantially improved.
## Added
- The Agent Passport drawer combines live route and session context with explicit profile switching, personality, model, reasoning, approval-policy, chat-override, and processing-tier controls.
- Secure browser-based Dashboard sign-in is scoped to the selected host. Chat, sessions, Manage, and Standard Voice share the same authenticated Gateway route while the API server remains an automatic fallback.
- Standard and Realtime voice settings use focused provider, model, and voice cards with upstream-aware discovery, descriptions, inline previews, waveforms, and a browsable catalog. Standard replies can begin speaking completed segments before generation finishes.
- User-started turns stay protected until every concurrent session settles. Privacy-safe notifications reopen the correct chat for approvals, questions, elevated permissions, or secure responses.
- Onboarding finishes with a layered permission review: notifications are recommended deliberately, optional capabilities remain separate, and users can continue without granting phone access.
- Chat surfaces one-turn model choices, approval modes, advisor progress, queued recovery, project labels, collapsible attachments, persisted images, interim Gateway events, and image-generation activity.
- Choose, preview, and install Petdex companions from Appearance, or import your own pet. Pets remain separate from agent avatars and the background Sphere.
- Hold and drag a pet anywhere, or enable optional UI-aware roaming across measured chat bubbles, the composer, settings cards, and other safe ledges.
- Open native Android pages contributed by installed Hermes plugins. Pages use a host-rendered declarative schema, and scoped writes remain disabled until explicitly granted.
- Use Relay 1.5.0 to review, keep, or remove approval-gated agent-created plugin-page drafts.
- Select Hermes as Android’s default Digital Assistant, with an optional local “Hey Hermes” listener that keeps pre-activation audio on the device.
- Use the complete AI-assisted Russian Android catalog from the in-app language picker.
## Improved
- Assistant and floating Voice controls start compact, expand for transcript and response detail, and hand off to full Voice without restarting the turn.
- Voice interruption now covers generation and playback with upstream-aligned calibration, stop phrases, and private next-turn interruption context.
- The Agent Passport presents profile configuration, skills, routing, reasoning, and scoped credentials more clearly.
- Pet roaming follows measured terrain, bubble edges, scroll movement, obstacle recovery, animation capabilities, temperament, and reduced-motion/accessibility pauses.
## Fixed
- Gateway reconnects reactivate the original live session without resubmitting acknowledged prompts or duplicating session rows.
- Tailscale, QR, and other remote routes move Dashboard, Gateway, sessions, Manage, Standard Voice, API fallback, and optional Relay together.
- Dashboard authentication, model routing, recovery, and profile state stay scoped to the selected connection and session, including during cold start and rapid switching.
- Promoted voice and background tasks keep their owning Chat row until the work settles.
- User-installed certificate authorities work for self-hosted HTTPS/WSS while normal chain, hostname, and Relay-pin verification remain enforced.
- Malformed syntax-highlighting ranges no longer crash Markdown rendering.
- Developer Options no longer exposes the obsolete Relay feature flag; version-tap unlock, relock, backup, import, and reset actions now persist and report accurately.
- Streaming voice output falls back when no first audio arrives, and long Standard Voice recordings upload without duplicate in-memory encoding.
- Relay failover no longer allows competing reconnect loops to bounce rapidly between LAN and remote routes.
- Streamed chat rows retain stable UI identity while server IDs and background-process state reconcile.
- Pets recover from occupied or scrolling terrain, avoid text and the jump-to-latest control, and preserve walk, jump, held, and drop animation states.
- OEM assistant picker activation, local wake completion, and empty-speech recovery are reliable across the supported lifecycle.
## Install / Verify
- App version: **1.5.0** (versionCode **33**).
- App version: **1.6.0** (versionCode **37**).
- Standard Chat and Vanilla Hermes voice continue to work against unmodified upstream Hermes.
- Petdex installation is built into Android. Native pages from ordinary installed plugins use the authenticated Dashboard; agent-created page drafts require the optional Relay 1.5.0 plugin.
+56 -19
View File
@@ -6,6 +6,24 @@ For shipped work, see `DEVLOG.md`. For architectural decisions, see `docs/decisi
---
## Android Plugin Studio protocol follow-ups
The first live declarative Plugin lane is host-local: Relay tools create bounded
draft JSON, Android previews it through the authenticated Dashboard namespace,
and exact-digest Keep/Remove actions require an Android user tap. Complete the
multi-session protocol before treating `lifecycle=session` as an isolation claim:
- Derive draft ownership from trusted Hermes task context and store only an HMAC
of that identifier; never accept a model-supplied session owner.
- Filter draft discovery by the Android app's active Hermes session while keeping
profile and connection publications separate with explicit precedence.
- Replace foreground five-second catalog polling with authenticated catalog
invalidation events plus ETag polling fallback.
- Expire abandoned drafts and pending approvals, and add revision-bound profile
versus connection promotion targets.
---
## Verify Android native dashboard sign-in on device
Android now selects Custom Tab + PKCE for HTTPS gateways that advertise
@@ -717,9 +735,10 @@ Deferred:
A 5-agent audit compared the chat surface to Discord/Telegram/Messenger/iMessage/
GitHub-mobile. **Shipped this pass (pending on-device verification):** a chat-tuned
`markdownTypography()` ramp (headings were falling through to M3 display roles —
h1=`displayLarge` 57sp in this app's scale — so a `#` was a billboard; now h1≈20sp
scaling down, list/paragraph unified to 14sp, inline+fenced code 13sp, `textLink`
`markdownTypography()` ramp (headings were falling through to M3 display roles —
h1=`displayLarge` 57sp in this app's scale — so a `#` was a billboard; now h1≈20sp
scaling down, list/paragraph unified to 15sp/21sp, primary assistant prose moved
to the theme's full-contrast `onSurface`, inline+fenced code 13sp, `textLink`
accent+underline) in `MarkdownContent.kt`; timestamp gated to `isLastInGroup` (was on
every bubble) + grouping breaks on a >5min gap (`GROUP_GAP_MS`) so a resumed
conversation gets its own beat; long-press haptic on the action menu; streaming dots
@@ -731,10 +750,6 @@ gated to pre-first-token. Deferred:
parses one full CommonMark document so global link references, indentation, and
nested containers remain correct; the viewport now anchors that same remeasure.
Verify lists, tables, quotes, HTML, nested fences, and reference links on-device.
- **Bubble body 14sp → 15sp/21.** 14sp is the smallest body of the five reference
apps. Bump markdown paragraph/text/list + the two plain `Text` sites
(`MessageBubble.kt` user/system) together; keep ~1.4 leading so the ~272dp measure
stays ~36–38 chars/line. Debatable/broad — left out of the certain heading win.
- **Tail-corner on last-in-group only (design decision).** The audit flagged the
per-bubble bottom tail as "half-implemented," but it's a deliberate aesthetic
(every bubble tails). Switching to iMessage-style "tail on the last bubble only"
@@ -1045,12 +1060,14 @@ to tool state, safety prompts, or the current task.
playback-synchronized amplitude through `shouldMarkRealtimeOutputActive`,
matching the basic-TTS path. Confirm visually on-device with the 1.4.1 batch.
- **Voice command layer — initial 1.4.1 subset code-complete; live verify and
navigation residuals remain.** Exact final transcripts can stop speech,
- **Voice command layer — upstream stop phrases and phase-aware pause are
code-complete; live verify and navigation residuals remain.** Exact final transcripts can end the active voice chat,
explicitly cancel the active background task, pause/resume Continuous mode,
repeat a settled background answer, and start a new Standard chat. Bare `stop`
and `cancel`, partial transcripts, and command-like ordinary prompts stay on the
normal Hermes route. Realtime `new chat` remains gated on a clean websocket
repeat a settled background answer, and start a new Standard chat. Bare
`stop` is configurable and exact-only while voice chat is active; bare
`pause` remains phase-gated to Continuous mode. `cancel`, partial transcripts,
and command-like ordinary prompts stay on the normal Hermes route. Realtime
`new chat` remains gated on a clean websocket
session-rebind boundary; `open overlay` and `return to Hermes` remain future
navigation commands. Verify barge-in Stop, pause during a background run, local
command Chat cleanup, and Continuous rearm on device.
@@ -1085,11 +1102,34 @@ and whether the agent is waiting on the user.
experimental barge-in choice. Relay update is server-first; local Voice/barge-in
values share one DataStore transaction, with relay rollback on local failure.
- **Barge-in hardening** — keep barge-in experimental until echo/self-recording
- **Barge-in hardening — code complete; on-device matrix remains.** Full-turn
listener ownership, AEC/noise suppression, upstream-compatible RMS
calibration and thresholds, configurable playback grace, duck/cut behavior,
late-delta fencing, next-turn interruption context, and single-microphone
handoff are implemented. Phone testing still needs to cover speakerphone/headphones, quiet/noisy rooms, Standard/Realtime
generation and playback, stop/pause, and resume-after-interruption.
is solved. The target path is proper AEC, playback-ducking, and a rule that
- **Experimental wake word — on-device validation.** Verify first-enable model
installation and integrity failure recovery, all supported ABIs, Android
notification/microphone permission variants, background-start restrictions,
task recreation from the detection notification, acoustic false-positive and
false-negative rates, battery impact, stop action, and wake→voice→wake
microphone handoff. Voice settings now provide a bounded real-microphone/model
test with an input meter; use it to distinguish audio capture from KWS tuning
before testing the full activation flow. The first release remains fixed to
“Hey Hermes”; do not
expose profile-specific phrases until routing and acoustic behavior are
implemented and validated.
output audio can never become a user turn.
- **Android Digital Assistant — on-device validation.** On a physical device,
select and remove Hermes through the system Assistant role; verify gesture,
power-button, screen-off, credential-lock, and unlocked “Hey Hermes”
invocation; confirm the system session appears without overlay/full-screen
permissions; exercise compact, expanded, collapsed, and full-Voice handoff
states, background tap-through, rotation and insets, cancel/back, microphone
denial, network failure, process kill/recreation, and wake→voice→wake
resumption. Measure idle battery drain because third-party assistants do not
receive Google's dedicated low-power hotword hardware.
- **Audio quality guardrails** — normalize output volume across realtime and
@@ -1221,11 +1261,8 @@ Follow-ups:
- **Sphere-skin parity.** Skins are still process-scoped + `adb push` only — the live tick and the importer cover pets, not skins. Extend the tick to `loadUserSkins` and add a `.json` skin import if hot-loading/adding skins in-app is wanted.
- `**adb push` into `Android/data` hangs on Samsung scoped storage.** Confirmed: pushing a pet pack to `/sdcard/Android/data/<pkg>/files/pets/` stalls (no bytes written) although `adb shell ls` of the dir works. In-app `.zip` import is the supported path; `/sdcard/Download` pushes fine. Consider softening `docs/pet-spec.md` + user-docs to lead with in-app import over adb.
- **On-device import/delete smoke.** Import `/sdcard/Download/lucy.zip` via Add a pet → confirm Lucy appears, selects, and animates all states; then remove it and confirm the avatar falls back to the Sphere.
- **Pet state-change re-decode can flash one blank frame.** When the agent state switches clips, the first frame of the new clip may briefly be blank during decode; prewarm/hold-last-frame to smooth it. Root cause is the same as the next item: `PetAvatar.Render` re-decodes from disk on every clip change.
- **Pet frame-sequence memory: no cap or downsample (audit 2026-06-19).** `decodeClip` decodes every frame of the selected clip into `List<ImageBitmap>` at full resolution with no `inSampleSize` downscale to the display size and no frame-count/dimension ceiling — a long sequence of large PNGs can use a lot of RAM and a single very large image can OOM `BitmapFactory`. Add `inSampleSize` downsampling to the avatar's draw size and/or a documented hard cap. Spec now warns authors (prefer sprite sheets), but the renderer doesn't enforce it.
- **Pet decoded-clip cache (audit 2026-06-19).** `PetAvatar.Render` keys `produceState` on `clip`, so idle→thinking→speaking→idle within one turn re-runs `BitmapFactory.decodeFile` from disk each transition (repeated I/O + GC churn, and the blank-frame flash above). Add a small per-avatar `Map<SphereState, PetFrames>` decode cache.
- **Pet behavior model — richer state association (spec'd 2026-06-19, `docs/pet-spec.md` "Agent states &amp; pet behavior").** Shipped: the honesty clamp (declared reactivity ∩ `PET_RENDERER_CAPABILITIES`), the friendly `writing` alias, the `**working`/tool-use overlay** (pet-local sub-state from `toolCallBurst`; opt-in `working` clip drives both the swap and the Tools badge), the **one-shot reaction layer** (`greet`/`wake` on appear, `done`/`celebrate` on turn-finish — opt-in, play-once-then-revert, transition-derived; `ONE_SHOT_MAX_MS` backstop), and `**intensity` modulation** (opt-in `reactive.intensity` → live playback speedup ≤1.6× via `rememberUpdatedState`; un-clamps the Activity badge). Voice · Tools · Activity reactivity is now complete. Remaining:
- `**attention` one-shot (only deferred behavior).** A reaction on notification arrival — needs a host event the avatar doesn't yet receive (unlike `greet`/`done`, which ride state transitions). Would plumb a notification edge into `AvatarRenderState` (or a side channel) + a `PetOneShot.Attention`. Low priority: the avatar is rarely on-screen when notifications land (backgrounded) — see the value analysis; revisit only if the avatar becomes an always-on surface (persistent overlay / Quest port).
- **On-device verification (working + one-shots + intensity).** Best seen in clean mode (`AgentTextFlow` feeds `toolCallBurst` + `streamingIntensity` + state transitions). Confirm: a `working` clip swaps in during a tool run and releases ~600ms after (`WORKING_BURST_THRESHOLD` 0.5); a `done` clip plays once on reply completion then returns to idle; a `greet` clip plays once when the avatar appears; with `intensity:true`, a writing/working loop visibly quickens while streaming. Watch for the known clip re-decode flash on each swap (separate TODO — decoded-clip cache).
- **On-device verification (working + one-shots + intensity).** Best seen in clean mode (`AgentTextFlow` feeds `toolCallBurst` + `streamingIntensity` + state transitions). Confirm: a `working` clip swaps in during a tool run and releases ~600ms after (`WORKING_BURST_THRESHOLD` 0.5); a `done` clip plays once on reply completion then returns to idle; a `greet` clip plays once when the avatar appears; with `intensity:true`, a writing/working loop visibly quickens while streaming. Confirm each decoded clip swap holds the previous complete visual until the new state is ready.
- **Undecodable-but-present image appears valid (audit 2026-06-19).** A file that exists but isn't a decodable image passes the loader's `isFile` check, so the pet shows in the picker but renders blank. Documented as a caveat; consider a cheap header sniff at load time if false-valid pets become a support issue.
+23 -1
View File
@@ -37,7 +37,7 @@ android {
// exempt from Play's 14-day closed-testing rule. See RELEASE.md.
applicationId = "com.axiomlabs.hermesrelay"
minSdk = 26
targetSdk = 35
targetSdk = 36
versionCode = libs.versions.appVersionCode.get().toInt()
versionName = libs.versions.appVersionName.get()
@@ -125,6 +125,7 @@ android {
}
release {
isMinifyEnabled = true
isShrinkResources = true
ndk {
debugSymbolLevel = "SYMBOL_TABLE"
}
@@ -164,6 +165,21 @@ android {
}
}
packaging {
jniLibs {
// sherpa-onnx v1.13.4 and the Silero VAD both use ONNX Runtime.
// Keep them on sherpa's 1.27.0 baseline and package one shared core.
pickFirsts += "**/libonnxruntime.so"
// The Android app calls only sherpa's JNI facade. These native C/C++
// API facades are development surfaces and are not loaded by the app.
excludes += setOf(
"**/libsherpa-onnx-c-api.so",
"**/libsherpa-onnx-cxx-api.so",
)
}
}
// JVM unit tests run against the stubbed Android SDK jar, where every
// platform API method throws RuntimeException("... not mocked") by
// default. With returnDefaultValues = true, those stubs instead
@@ -262,6 +278,12 @@ dependencies {
// Bundled ONNX Silero model (~2.2 MB); pulled from JitPack.
implementation(libs.android.vad.silero)
// Experimental, opt-in local keyword spotting. Models are downloaded only
// after the user enables the feature; no model binary is bundled in APKs.
// Keep the shared runtime aligned with sherpa-onnx v1.13.4.
implementation(libs.onnxruntime.android)
implementation(libs.sherpa.onnx)
// Google Play In-App Update — googlePlay flavor ONLY (FLEXIBLE flow).
// Scoped via the `googlePlayImplementation` configuration so it never
// ships in the sideload APK, which updates via the GitHub-releases
@@ -0,0 +1,70 @@
package com.hermesandroid.relay.plugins.ui
import androidx.compose.material3.MaterialTheme
import androidx.compose.ui.test.assertIsDisplayed
import androidx.compose.ui.test.isToggleable
import androidx.compose.ui.test.junit4.createComposeRule
import androidx.compose.ui.test.onNodeWithText
import androidx.compose.ui.test.performClick
import com.hermesandroid.relay.plugins.document.PluginDocumentState
import com.hermesandroid.relay.plugins.document.PluginElement
import com.hermesandroid.relay.plugins.document.PluginPage
import com.hermesandroid.relay.plugins.document.PluginText
import com.hermesandroid.relay.plugins.document.PluginValue
import org.junit.Assert.assertEquals
import org.junit.Rule
import org.junit.Test
class PluginDocumentRendererTest {
@get:Rule
val composeTestRule = createComposeRule()
@Test
fun pageRendersBindingsAndEmitsControlledStateChanges() {
var interaction: PluginInteraction? = null
val page = PluginPage(
id = "home",
title = PluginText.Binding("title", "Fallback"),
content = PluginElement.Group(
id = "root",
children = listOf(
PluginElement.Text(
id = "message",
text = PluginText.Binding("message"),
),
PluginElement.Toggle(
id = "enabled-toggle",
label = PluginText.Literal("Enabled"),
binding = "enabled",
),
),
),
)
val state = PluginDocumentState(
mapOf(
"title" to PluginValue.StringValue("Status plugin"),
"message" to PluginValue.StringValue("Everything is healthy"),
"enabled" to PluginValue.BooleanValue(false),
),
)
composeTestRule.setContent {
MaterialTheme {
PluginPageRenderer(page, state, { interaction = it })
}
}
composeTestRule.onNodeWithText("Status plugin").assertIsDisplayed()
composeTestRule.onNodeWithText("Everything is healthy").assertIsDisplayed()
composeTestRule.onNode(isToggleable()).performClick()
assertEquals(
PluginInteraction.ValueChanged(
elementId = "enabled-toggle",
key = "enabled",
value = PluginValue.BooleanValue(true),
),
interaction,
)
}
}
@@ -0,0 +1,147 @@
package com.hermesandroid.relay.ui.components
import androidx.compose.foundation.layout.Box
import androidx.compose.material3.MaterialTheme
import androidx.compose.runtime.Composable
import androidx.compose.runtime.CompositionLocalProvider
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.testTag
import androidx.compose.ui.test.assertDoesNotExist
import androidx.compose.ui.test.assertExists
import androidx.compose.ui.test.junit4.createComposeRule
import androidx.compose.ui.test.onNodeWithContentDescription
import androidx.compose.ui.test.onNodeWithTag
import androidx.compose.ui.test.onNodeWithText
import androidx.test.platform.app.InstrumentationRegistry
import com.hermesandroid.relay.R
import com.hermesandroid.relay.ui.components.avatar.AgentAvatar
import com.hermesandroid.relay.ui.components.avatar.AvatarRenderState
import com.hermesandroid.relay.ui.components.avatar.AvatarSource
import com.hermesandroid.relay.ui.components.avatar.LocalAgentAvatar
import com.hermesandroid.relay.ui.components.avatar.LocalBackgroundVisualizationEnabled
import com.hermesandroid.relay.viewmodel.InteractionMode
import com.hermesandroid.relay.viewmodel.VoiceState
import com.hermesandroid.relay.viewmodel.VoiceUiState
import org.junit.Rule
import org.junit.Test
class AmbientVisualizationVisibilityTest {
@get:Rule
val composeTestRule = createComposeRule()
@Test
fun cleanMode_backgroundOff_hidesSphereAndKeepsComposer() {
composeTestRule.setContent {
AmbientTestProviders(enabled = false) {
CleanChatMode(
messages = emptyList(),
isStreaming = false,
sphereState = SphereState.Idle,
streamingIntensity = 0f,
toolCallBurst = 0f,
animationEnabled = true,
enabled = true,
onSend = {},
onExit = {},
)
}
}
composeTestRule.onNodeWithTag(AMBIENT_RENDERER_TAG).assertDoesNotExist()
composeTestRule.onNodeWithContentDescription(targetString(R.string.agent_text_send_cd))
.assertExists()
}
@Test
fun cleanMode_backgroundOn_rendersSphere() {
composeTestRule.setContent {
AmbientTestProviders(enabled = true) {
CleanChatMode(
messages = emptyList(),
isStreaming = false,
sphereState = SphereState.Idle,
streamingIntensity = 0f,
toolCallBurst = 0f,
animationEnabled = false,
enabled = true,
onSend = {},
onExit = {},
)
}
}
composeTestRule.onNodeWithTag(AMBIENT_RENDERER_TAG).assertExists()
}
@Test
fun voiceMode_backgroundOff_hidesSphereAndKeepsVoiceUi() {
composeTestRule.setContent {
AmbientTestProviders(enabled = false) {
TestVoiceOverlay()
}
}
composeTestRule.onNodeWithTag(AMBIENT_RENDERER_TAG).assertDoesNotExist()
composeTestRule.onNodeWithText(targetString(R.string.voice_overlay_tap_mic)).assertExists()
}
@Test
fun voiceMode_backgroundOn_rendersSphere() {
composeTestRule.setContent {
AmbientTestProviders(enabled = true) {
TestVoiceOverlay()
}
}
composeTestRule.onNodeWithTag(AMBIENT_RENDERER_TAG).assertExists()
}
@Composable
private fun AmbientTestProviders(enabled: Boolean, content: @Composable () -> Unit) {
MaterialTheme {
CompositionLocalProvider(
LocalAgentAvatar provides TaggedAmbientRenderer,
LocalBackgroundVisualizationEnabled provides enabled,
content = content,
)
}
}
@Composable
private fun TestVoiceOverlay() {
VoiceModeOverlay(
uiState = VoiceUiState(
voiceMode = true,
state = VoiceState.Idle,
interactionMode = InteractionMode.TapToTalk,
),
onMicTap = {},
onMicRelease = {},
onInterrupt = {},
onDismiss = {},
onModeChange = {},
onClearError = {},
)
}
private fun targetString(id: Int): String =
InstrumentationRegistry.getInstrumentation().targetContext.getString(id)
private object TaggedAmbientRenderer : AgentAvatar {
override val id = "ambient-test"
override val label = "Ambient test"
override val description = "Test renderer"
override val source = AvatarSource.BUILT_IN
override val reactivity = SphereReactivity()
@Composable
override fun Render(state: AvatarRenderState, modifier: Modifier) {
Box(modifier = modifier.testTag(AMBIENT_RENDERER_TAG))
}
}
private companion object {
const val AMBIENT_RENDERER_TAG = "ambientVisualizationRenderer"
}
}
@@ -1 +1 @@
Connect through secure Dashboard sign-in, switch profiles from the new Agent Passport, and keep multiple background chats active with actionable approval and question alerts. Image generation, attachments, model routing, voice, and Gateway recovery are clearer and more reliable. Setup now guides optional notification, camera, microphone, and companion permissions without blocking chat.
Add floating Petdex companions that can roam across the interface, safe native pages from installed Hermes plugins, and an optional Android Digital Assistant with local “Hey Hermes.” This release also adds Russian and improves voice recovery, route failover, live chat stability, and pet movement.
@@ -1 +1 @@
通过安全的 Dashboard 登录连接,并在新的智能体护照中切换配置文件。多个后台对话可保持运行,审批或提问通知可直接返回正确会话。图像生成、附件、模型路由、语音和 Gateway 恢复更加清晰可靠。设置流程会说明可选的通知、相机、麦克风和通知伴侣权限,且不会阻止聊天。
新增可在界面中漫游的 Petdex 浮动宠物、由已安装 Hermes 插件提供的安全原生页面,以及支持本地“Hey Hermes”的可选 Android 数字助理。本次更新还新增俄语,并改进语音恢复、路线切换、实时聊天稳定性和宠物移动。
+70
View File
@@ -20,6 +20,7 @@
for the device-control bridge service; the merger dedups.) -->
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
<uses-feature android:name="android.hardware.camera" android:required="false" />
@@ -47,6 +48,13 @@
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
<!-- Some Android OEM assistant pickers enumerate ACTION_ASSIST
activities in addition to VoiceInteractionService providers. -->
<intent-filter>
<action android:name="android.intent.action.ASSIST" />
<category android:name="android.intent.category.DEFAULT" />
<category android:name="android.intent.category.VOICE" />
</intent-filter>
</activity>
<!-- AppCompat persists in-app language choices on Android 12 and lower.
@@ -104,6 +112,68 @@
android:value="Keeps user-started Hermes turns connected until they finish or need input, and optionally keeps idle connections responsive when the user enables Persistent connection." />
</service>
<!-- Experimental, explicitly user-started on-device wake-word listener.
Audio remains local and the service is never boot/restart started. -->
<service
android:name=".wake.WakeWordForegroundService"
android:exported="false"
android:foregroundServiceType="microphone"
android:stopWithTask="false" />
<!-- User-started protection for voice capture from the system overlay.
The service does not own AudioRecord; it keeps foreground-only
microphone app-ops available while Hermes is behind another app. -->
<service
android:name=".voice.VoiceOverlayForegroundService"
android:exported="false"
android:foregroundServiceType="microphone"
android:stopWithTask="false" />
<!-- Explicitly opt-in Android Digital Assistant integration. Android
binds this only after the user selects Hermes for ROLE_ASSISTANT. -->
<service
android:name=".assistant.HermesVoiceInteractionService"
android:exported="true"
android:label="@string/assistant_service_label"
android:permission="android.permission.BIND_VOICE_INTERACTION">
<intent-filter>
<action android:name="android.service.voice.VoiceInteractionService" />
</intent-filter>
<meta-data
android:name="android.voice_interaction"
android:resource="@xml/voice_interaction_service" />
</service>
<!-- Heavy assistant UI is isolated from the always-running interaction
service, matching the platform lifecycle guidance. -->
<service
android:name=".assistant.HermesVoiceInteractionSessionService"
android:exported="true"
android:permission="android.permission.BIND_VOICE_INTERACTION"
android:process=":assistant_session" />
<!-- Required companion component for VoiceInteractionService metadata.
Hermes session transcription remains owned by the existing voice
pipeline; this service does not open a second microphone stream. -->
<service
android:name=".assistant.HermesRecognitionService"
android:exported="true"
android:permission="android.permission.BIND_VOICE_INTERACTION">
<intent-filter>
<action android:name="android.speech.RecognitionService" />
<category android:name="android.intent.category.DEFAULT" />
</intent-filter>
</service>
<receiver
android:name=".assistant.AssistantSessionStateReceiver"
android:exported="false"
android:process=":assistant_session" />
<receiver
android:name=".assistant.AssistantSessionLifecycleReceiver"
android:exported="false" />
</application>
</manifest>
+96
View File
@@ -1,5 +1,101 @@
{
"versions": [
{
"version": "1.6.0",
"title": "Pets, plugins, and voice",
"date": "2026-08-02",
"sections": [
{
"header": "A companion with personality",
"bullets": [
"Browse and install Petdex companions, or import your own pet without replacing the agent avatar or background Sphere.",
"Drag a pet anywhere or let it roam across measured chat bubbles, settings cards, controls, and other safe UI ledges."
]
},
{
"header": "Native plugin pages",
"bullets": [
"Installed Hermes plugins can contribute host-rendered native pages without loading executable plugin code on the phone.",
"Scoped writes stay off until granted, while Relay 1.5.0 adds approval-gated agent-created page previews."
]
},
{
"header": "Hermes as your assistant",
"bullets": [
"Optionally select Hermes as Android’s Digital Assistant and use a local “Hey Hermes” listener for background or locked-screen sessions.",
"Compact assistant and floating Voice controls expand for detail and continue the same turn when full Voice opens."
]
},
{
"header": "More reliable everywhere",
"bullets": [
"Voice output recovery, long recordings, route failover, streamed chat identity, and pet terrain recovery are more resilient.",
"Android now includes a complete AI-assisted Russian catalog refreshed for the 1.6 feature set."
]
}
]
},
{
"version": "1.5.3",
"title": "Voice stays open",
"date": "2026-07-31",
"sections": [
{
"header": "Stable voice transcripts",
"bullets": [
"Voice Focus keeps stable transcript rows while live messages reconcile with persisted chat history, preventing duplicate-key crashes that could close the app."
]
}
]
},
{
"version": "1.5.2",
"title": "Sign in without detours",
"date": "2026-07-28",
"sections": [
{
"header": "Provider-compatible sign-in",
"bullets": [
"Self-hosted OIDC returns through the dashboard callback, while Nous Portal opens securely in the system browser.",
"Private-LAN and Tailscale dashboard routes preserve the configured HTTPS callback and keep credentials scoped to the active connection."
]
},
{
"header": "Stable conversation updates",
"bullets": [
"Replayed upstream chat events are coalesced before rendering so duplicate message identifiers do not destabilize the conversation list."
]
}
]
},
{
"version": "1.5.1",
"title": "Voice and chat stay in place",
"date": "2026-07-26",
"sections": [
{
"header": "Voice at the right depth",
"bullets": [
"Use Voice Focus for a compact spoken-turn view or Conversation for the complete Chat renderer without leaving the active voice session.",
"Keep intermediate work visual while supported voice paths wait to speak the settled final response."
]
},
{
"header": "Reliable narration and background work",
"bullets": [
"Standard Voice now speaks valid completed replies after generation hands off to narration.",
"Realtime background tasks release foreground voice controls while their progress and results remain reachable."
]
},
{
"header": "Formatted answers stay readable",
"bullets": [
"Completed streams render headings, lists, emphasis, and code blocks without returning to the beginning of the answer.",
"Assistant text uses stronger theme contrast and a more comfortable chat reading scale."
]
}
]
},
{
"version": "1.5.0",
"title": "Hermes, always in reach",
+5 -5
View File
@@ -1,6 +1,6 @@
v1.5.0 - Hermes, always in reach
v1.6.0 - Pets, plugins, and voice
* Connect through secure Dashboard sign-in and switch profiles from the new Agent Passport.
* Keep multiple background chats active and reopen the right session from approval or question alerts.
* Follow richer attachments, image generation, model routing, Gateway recovery, and streaming voice.
* Finish setup with clear, optional permission guidance that never blocks standard chat.
* Add floating Petdex companions that can roam across the interface.
* Add safe native pages contributed by installed Hermes plugins.
* Use Hermes as the optional Android Digital Assistant with local “Hey Hermes.”
* Add Russian and improve voice, routing, chat, and pet stability.
@@ -1,6 +1,9 @@
package com.hermesandroid.relay
import android.app.ActivityManager
import android.app.Application
import android.content.Context
import android.os.Build
import coil3.ImageLoader
import coil3.PlatformContext
import coil3.SingletonImageLoader
@@ -9,11 +12,24 @@ import coil3.request.crossfade
import com.hermesandroid.relay.bridge.UnattendedAccessManager
import com.hermesandroid.relay.data.AppAnalytics
import com.hermesandroid.relay.power.WakeLockManager
import com.hermesandroid.relay.runtime.HermesProcessRuntime
import com.hermesandroid.relay.util.AppForegroundTracker
import com.hermesandroid.relay.util.CrashReporter
class HermesRelayApp : Application(), SingletonImageLoader.Factory {
/**
* Shared chat/voice runtime for the main application process. It is lazy so
* the always-available assistant session UI process stays lightweight and
* cannot accidentally become a second microphone/session owner.
*/
val runtime: HermesProcessRuntime by lazy(LazyThreadSafetyMode.SYNCHRONIZED) {
check(isMainApplicationProcess()) {
"HermesProcessRuntime may only be created in the main application process"
}
HermesProcessRuntime(this)
}
/**
* Coil's singleton image loader for the whole app. Registering the OkHttp
* network fetcher EXPLICITLY guarantees `http(s)` image URLs (e.g. a
@@ -51,6 +67,19 @@ class HermesRelayApp : Application(), SingletonImageLoader.Factory {
AppForegroundTracker.initialize()
}
private fun isMainApplicationProcess(): Boolean {
val processName = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
getProcessName()
} else {
val pid = android.os.Process.myPid()
val activityManager = getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager
activityManager.runningAppProcesses
?.firstOrNull { process -> process.pid == pid }
?.processName
}
return processName == packageName
}
companion object {
lateinit var instance: HermesRelayApp
private set
@@ -5,16 +5,18 @@ import android.content.Context
import android.content.Intent
import android.media.projection.MediaProjectionManager
import android.os.Bundle
import android.os.Build
import android.util.Log
import android.view.View
import android.view.WindowManager
import android.view.animation.DecelerateInterpolator
import androidx.activity.compose.setContent
import androidx.activity.enableEdgeToEdge
import androidx.activity.result.contract.ActivityResultContracts
import androidx.activity.viewModels
import androidx.core.animation.doOnEnd
import androidx.core.splashscreen.SplashScreen.Companion.installSplashScreen
import androidx.appcompat.app.AppCompatActivity
import androidx.lifecycle.lifecycleScope
import com.hermesandroid.relay.accessibility.ScreenCaptureRequester
import com.hermesandroid.relay.bridge.BridgeForegroundService
import com.hermesandroid.relay.bridge.UnattendedAccessManager
@@ -24,10 +26,13 @@ import com.hermesandroid.relay.notifications.InteractionRequestNotifier
import com.hermesandroid.relay.ui.RelayApp
import com.hermesandroid.relay.util.NavRouteRequest
import com.hermesandroid.relay.viewmodel.ConnectionViewModel
import kotlinx.coroutines.launch
import kotlinx.coroutines.flow.collect
class MainActivity : AppCompatActivity() {
private val connectionViewModel: ConnectionViewModel by viewModels()
private val connectionViewModel: ConnectionViewModel
get() = (applicationContext as HermesRelayApp).runtime.connectionViewModel
// === PHASE3-bridge-ui-followup: MediaProjection consent flow ===
// ActivityResultLauncher for the system screen-capture consent dialog.
@@ -67,6 +72,8 @@ class MainActivity : AppCompatActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
val splashScreen = installSplashScreen()
com.hermesandroid.relay.assistant.AssistantSessionProtocol
.prepareAssistActivation(intent)
// Hold splash until DataStore is loaded and onboarding status is known
splashScreen.setKeepOnScreenCondition {
@@ -88,6 +95,12 @@ class MainActivity : AppCompatActivity() {
}
super.onCreate(savedInstanceState)
configureAssistantWindow(intent)
lifecycleScope.launch {
com.hermesandroid.relay.assistant.AssistantAppSessionState.active.collect { active ->
if (!active) clearAssistantWindow()
}
}
enableEdgeToEdge()
// === PHASE3-bridge-ui-followup: install MediaProjection requester ===
@@ -114,6 +127,14 @@ class MainActivity : AppCompatActivity() {
// in RelayApp's NavRouteRequest collector — we just pump the request
// into the SharedFlow here.
consumeNavRouteIntent(intent)
val consumedAssistantActivation =
com.hermesandroid.relay.assistant.AssistantSessionProtocol.consumeActivation(
this,
intent,
)
if (!consumedAssistantActivation) {
com.hermesandroid.relay.assistant.AssistantSessionProtocol.restoreActivation(this)
}
// === END PHASE3-safety-rails-followup ===
setContent {
RelayApp()
@@ -122,6 +143,9 @@ class MainActivity : AppCompatActivity() {
override fun onNewIntent(intent: Intent) {
super.onNewIntent(intent)
com.hermesandroid.relay.assistant.AssistantSessionProtocol
.prepareAssistActivation(intent)
configureAssistantWindow(intent)
// === PHASE3-safety-rails-followup: deep-link nav route on re-launch ===
// Same as onCreate but for the singleTask / FLAG_ACTIVITY_CLEAR_TOP
// path: when the app is already running and the foreground service's
@@ -129,6 +153,7 @@ class MainActivity : AppCompatActivity() {
// instead of onCreate. RelayApp's collector handles both cases.
setIntent(intent)
consumeNavRouteIntent(intent)
com.hermesandroid.relay.assistant.AssistantSessionProtocol.consumeActivation(this, intent)
// === END PHASE3-safety-rails-followup ===
}
@@ -138,6 +163,40 @@ class MainActivity : AppCompatActivity() {
NavRouteRequest.tryRequest(route)
}
private fun configureAssistantWindow(intent: Intent?) {
if (
intent?.getBooleanExtra(
com.hermesandroid.relay.assistant.AssistantSessionProtocol.EXTRA_ASSISTANT_SESSION,
false,
) == true ||
com.hermesandroid.relay.assistant.AssistantSessionPersistence.isActive(this)
) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
setShowWhenLocked(true)
setTurnScreenOn(true)
} else {
@Suppress("DEPRECATION")
window.addFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
}
}
}
private fun clearAssistantWindow() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
setShowWhenLocked(false)
setTurnScreenOn(false)
} else {
@Suppress("DEPRECATION")
window.clearFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
}
}
override fun onResume() {
super.onResume()
// Returning to the app clears the one-slot "Hermes finished
@@ -0,0 +1,426 @@
package com.hermesandroid.relay.assistant
import android.app.role.RoleManager
import android.content.BroadcastReceiver
import android.content.ComponentName
import android.content.Context
import android.content.Intent
import android.os.Build
import android.provider.Settings
import android.service.voice.VoiceInteractionService
import androidx.core.content.edit
import com.hermesandroid.relay.viewmodel.VoiceState
import com.hermesandroid.relay.viewmodel.VoiceUiState
import com.hermesandroid.relay.HermesRelayApp
import com.hermesandroid.relay.wake.WakeWordActivation
import com.hermesandroid.relay.wake.WakeWordActivationCoordinator
import com.hermesandroid.relay.wake.WakeWordActivationSource
import com.hermesandroid.relay.wake.WakeWordProfileRouting
import java.util.UUID
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
enum class AssistantRoleStatus {
Unavailable,
NotSelected,
Selected,
}
enum class AssistantSessionPhase {
Launching,
Listening,
Transcribing,
Thinking,
Speaking,
Idle,
Error,
Closed,
}
data class AssistantSessionSnapshot(
val phase: AssistantSessionPhase = AssistantSessionPhase.Launching,
val transcript: String? = null,
val response: String = "",
val error: String? = null,
)
object AssistantRole {
fun status(context: Context): AssistantRoleStatus {
val component = ComponentName(context, HermesVoiceInteractionService::class.java)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
val roles = context.getSystemService(RoleManager::class.java)
?: return AssistantRoleStatus.Unavailable
if (!roles.isRoleAvailable(RoleManager.ROLE_ASSISTANT)) {
return AssistantRoleStatus.Unavailable
}
return if (roles.isRoleHeld(RoleManager.ROLE_ASSISTANT) &&
VoiceInteractionService.isActiveService(context, component)
) {
AssistantRoleStatus.Selected
} else {
AssistantRoleStatus.NotSelected
}
}
return if (VoiceInteractionService.isActiveService(context, component)) {
AssistantRoleStatus.Selected
} else {
AssistantRoleStatus.NotSelected
}
}
fun selectionIntent(context: Context): Intent? {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
val roles = context.getSystemService(RoleManager::class.java)
if (roles?.isRoleAvailable(RoleManager.ROLE_ASSISTANT) == true) {
return roles.createRequestRoleIntent(RoleManager.ROLE_ASSISTANT)
}
}
return Intent(Settings.ACTION_VOICE_INPUT_SETTINGS)
.takeIf { it.resolveActivity(context.packageManager) != null }
}
fun managementIntent(context: Context): Intent? =
Intent(Settings.ACTION_VOICE_INPUT_SETTINGS)
.takeIf { it.resolveActivity(context.packageManager) != null }
?: selectionIntent(context)
}
/**
* Cross-process protocol between the system-owned assistant session process
* and the normal app process that owns the established voice pipeline.
*/
object AssistantSessionProtocol {
const val EXTRA_ASSISTANT_SESSION = "com.hermesandroid.relay.assistant.SESSION"
const val EXTRA_ACTIVATION_ID = "com.hermesandroid.relay.assistant.ACTIVATION_ID"
const val EXTRA_START_NEW_SESSION =
"com.hermesandroid.relay.assistant.START_NEW_SESSION"
const val EXTRA_HANDOFF_ONLY = "com.hermesandroid.relay.assistant.HANDOFF_ONLY"
private const val ACTION_STATUS = "com.hermesandroid.relay.assistant.STATUS"
private const val ACTION_FINISH = "com.hermesandroid.relay.assistant.FINISH"
private const val ACTION_START = "com.hermesandroid.relay.assistant.START"
private const val ACTION_ACTIVATE = "com.hermesandroid.relay.assistant.ACTIVATE"
private const val EXTRA_PHASE = "phase"
private const val EXTRA_TRANSCRIPT = "transcript"
private const val EXTRA_RESPONSE = "response"
private const val EXTRA_ERROR = "error"
private const val EXTRA_CANCEL_VOICE = "cancel_voice"
fun prepareAssistActivation(intent: Intent?) {
val assistIntent = intent ?: return
if (!isAssistAction(assistIntent.action)) return
if (assistIntent.getBooleanExtra(EXTRA_HANDOFF_ONLY, false)) return
assistIntent.putExtra(EXTRA_ASSISTANT_SESSION, true)
}
internal fun isAssistAction(action: String?): Boolean = action == Intent.ACTION_ASSIST
fun activationIntent(
context: Context,
activationId: String = UUID.randomUUID().toString(),
startNewSession: Boolean = true,
) =
Intent(context, com.hermesandroid.relay.MainActivity::class.java).apply {
action = Intent.ACTION_ASSIST
putExtra(EXTRA_ASSISTANT_SESSION, true)
putExtra(EXTRA_ACTIVATION_ID, activationId)
putExtra(EXTRA_START_NEW_SESSION, startNewSession)
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP)
}
fun fullVoiceIntent(context: Context) =
Intent(context, com.hermesandroid.relay.MainActivity::class.java).apply {
action = Intent.ACTION_ASSIST
addCategory(Intent.CATEGORY_VOICE)
putExtra(EXTRA_HANDOFF_ONLY, true)
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP)
}
fun activate(
context: Context,
activationId: String = UUID.randomUUID().toString(),
startNewSession: Boolean = true,
) {
context.sendBroadcast(
Intent(context, AssistantSessionLifecycleReceiver::class.java).apply {
action = ACTION_ACTIVATE
putExtra(EXTRA_ACTIVATION_ID, activationId)
putExtra(EXTRA_START_NEW_SESSION, startNewSession)
}
)
}
fun consumeActivation(context: Context, intent: Intent?): Boolean {
if (intent?.getBooleanExtra(EXTRA_HANDOFF_ONLY, false) == true) {
intent.removeExtra(EXTRA_HANDOFF_ONLY)
com.hermesandroid.relay.util.NavRouteRequest.tryRequest("chat")
return true
}
if (intent?.getBooleanExtra(EXTRA_ASSISTANT_SESSION, false) != true) return false
val id = intent.getStringExtra(EXTRA_ACTIVATION_ID) ?: UUID.randomUUID().toString()
val startNewSession = intent.getBooleanExtra(EXTRA_START_NEW_SESSION, true)
AssistantSessionPersistence.setActivation(context, id, startNewSession)
WakeWordActivationCoordinator.request(
WakeWordActivation(
id = id,
startNewSession = startNewSession,
profileRouting = WakeWordProfileRouting(),
source = WakeWordActivationSource.SystemAssistant,
)
)
AssistantAppSessionState.setActive(true)
intent.removeExtra(EXTRA_ASSISTANT_SESSION)
intent.removeExtra(EXTRA_ACTIVATION_ID)
intent.removeExtra(EXTRA_START_NEW_SESSION)
return true
}
fun restoreActivation(context: Context): Boolean {
if (AssistantAppSessionState.active.value) return false
val activation = AssistantSessionPersistence.restoreActivation(context) ?: return false
AssistantAppSessionState.setActive(true)
HermesVoiceInteractionService.setVoiceSessionActive(true)
val application = context.applicationContext as HermesRelayApp
application.runtime.requestVoiceActivation(
activationId = activation.id,
startNewSession = activation.startNewSession,
onFailure = { failure ->
publish(
application,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
),
)
},
)
return true
}
fun publish(context: Context, snapshot: AssistantSessionSnapshot) {
context.sendBroadcast(
Intent(context, AssistantSessionStateReceiver::class.java).apply {
action = ACTION_STATUS
putExtra(EXTRA_PHASE, snapshot.phase.name)
putExtra(EXTRA_TRANSCRIPT, snapshot.transcript)
putExtra(EXTRA_RESPONSE, snapshot.response)
putExtra(EXTRA_ERROR, snapshot.error)
}
)
if (shouldFinishLifecycleOnSnapshot(snapshot)) {
// The session UI runs in a separate process. Reconcile the app-owned
// lifecycle directly as well so a reclaimed hidden UI process cannot
// leave wake listening paused after full Voice closes.
finish(context, cancelVoice = false)
}
}
fun publish(context: Context, state: VoiceUiState) {
publish(context, snapshotFromVoiceState(state))
}
internal fun snapshotFromVoiceState(state: VoiceUiState): AssistantSessionSnapshot {
val phase = when {
!state.voiceMode -> AssistantSessionPhase.Closed
state.state == VoiceState.Listening -> AssistantSessionPhase.Listening
state.state == VoiceState.Transcribing -> AssistantSessionPhase.Transcribing
state.state == VoiceState.Thinking -> AssistantSessionPhase.Thinking
state.state == VoiceState.Speaking -> AssistantSessionPhase.Speaking
state.state == VoiceState.Error -> AssistantSessionPhase.Error
else -> AssistantSessionPhase.Idle
}
return AssistantSessionSnapshot(
phase = phase,
transcript = state.transcribedText?.take(MAX_SESSION_TEXT_CHARS),
response = state.responseText.take(MAX_SESSION_TEXT_CHARS),
error = state.error?.take(MAX_SESSION_ERROR_CHARS),
)
}
internal fun shouldFinishLifecycleOnSnapshot(snapshot: AssistantSessionSnapshot): Boolean =
snapshot.phase == AssistantSessionPhase.Closed
fun finish(context: Context, cancelVoice: Boolean) {
context.sendBroadcast(
Intent(context, AssistantSessionLifecycleReceiver::class.java).apply {
action = ACTION_FINISH
putExtra(EXTRA_CANCEL_VOICE, cancelVoice)
}
)
}
fun started(context: Context) {
context.sendBroadcast(
Intent(context, AssistantSessionLifecycleReceiver::class.java).setAction(ACTION_START)
)
}
internal fun isFinishAction(action: String?): Boolean = action == ACTION_FINISH
internal fun isStartAction(action: String?): Boolean = action == ACTION_START
internal fun isActivateAction(action: String?): Boolean = action == ACTION_ACTIVATE
internal fun shouldCancelVoice(intent: Intent): Boolean =
intent.getBooleanExtra(EXTRA_CANCEL_VOICE, false)
internal fun readSnapshot(intent: Intent): AssistantSessionSnapshot {
val phase = runCatching {
AssistantSessionPhase.valueOf(
intent.getStringExtra(EXTRA_PHASE) ?: AssistantSessionPhase.Launching.name
)
}.getOrDefault(AssistantSessionPhase.Error)
return AssistantSessionSnapshot(
phase = phase,
transcript = intent.getStringExtra(EXTRA_TRANSCRIPT),
response = intent.getStringExtra(EXTRA_RESPONSE).orEmpty(),
error = intent.getStringExtra(EXTRA_ERROR),
)
}
private const val MAX_SESSION_TEXT_CHARS = 4_000
private const val MAX_SESSION_ERROR_CHARS = 1_000
}
object AssistantSessionState {
private val _snapshot = MutableStateFlow(AssistantSessionSnapshot())
val snapshot: StateFlow<AssistantSessionSnapshot> = _snapshot.asStateFlow()
internal fun update(snapshot: AssistantSessionSnapshot) {
_snapshot.value = snapshot
}
internal fun reset() {
_snapshot.value = AssistantSessionSnapshot()
}
}
class AssistantSessionStateReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
AssistantSessionState.update(AssistantSessionProtocol.readSnapshot(intent))
}
}
class AssistantSessionLifecycleReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
if (AssistantSessionProtocol.isActivateAction(intent.action)) {
val id = intent.getStringExtra(AssistantSessionProtocol.EXTRA_ACTIVATION_ID)
?: UUID.randomUUID().toString()
val startNewSession = intent.getBooleanExtra(
AssistantSessionProtocol.EXTRA_START_NEW_SESSION,
true,
)
AssistantSessionPersistence.setActive(context, true)
AssistantSessionPersistence.setActivation(context, id, startNewSession)
AssistantAppSessionState.setActive(true)
HermesVoiceInteractionService.setVoiceSessionActive(true)
val application = context.applicationContext as HermesRelayApp
// Dispatch into the process-owned scope and return from the receiver
// immediately. Cold readiness can take longer than a broadcast's
// execution budget.
application.runtime.requestVoiceActivation(
activationId = id,
startNewSession = startNewSession,
onFailure = { failure ->
AssistantSessionProtocol.publish(
application,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
),
)
},
)
return
}
if (AssistantSessionProtocol.isStartAction(intent.action)) {
AssistantSessionPersistence.setActive(context, true)
HermesVoiceInteractionService.setVoiceSessionActive(true)
return
}
if (!AssistantSessionProtocol.isFinishAction(intent.action)) return
AssistantSessionPersistence.setActive(context, false)
if (AssistantSessionProtocol.shouldCancelVoice(intent)) {
val application = context.applicationContext as HermesRelayApp
application.runtime.cancelVoice()
}
AssistantAppSessionState.setActive(false)
HermesVoiceInteractionService.setVoiceSessionActive(false)
}
}
object AssistantSessionPersistence {
private const val STORE = "assistant_session_lifecycle"
private const val KEY_ACTIVE_SINCE = "active_since"
private const val KEY_ACTIVATION_ID = "activation_id"
private const val KEY_START_NEW_SESSION = "start_new_session"
private const val STALE_AFTER_MS = 30 * 60 * 1_000L
fun setActive(context: Context, active: Boolean) {
context.getSharedPreferences(STORE, Context.MODE_PRIVATE).edit(commit = true) {
putLong(KEY_ACTIVE_SINCE, if (active) System.currentTimeMillis() else 0L)
if (!active) {
remove(KEY_ACTIVATION_ID)
}
}
}
fun setActivation(context: Context, id: String, startNewSession: Boolean) {
context.getSharedPreferences(STORE, Context.MODE_PRIVATE).edit(commit = true) {
putString(KEY_ACTIVATION_ID, id)
putBoolean(KEY_START_NEW_SESSION, startNewSession)
}
}
fun restoreActivation(context: Context): WakeWordActivation? {
if (!isActive(context)) return null
val store = context.getSharedPreferences(STORE, Context.MODE_PRIVATE)
val id = store.getString(KEY_ACTIVATION_ID, null) ?: return null
return WakeWordActivation(
id = id,
startNewSession = store.getBoolean(KEY_START_NEW_SESSION, true),
profileRouting = WakeWordProfileRouting(),
source = WakeWordActivationSource.SystemAssistant,
)
}
fun isActive(context: Context, nowMs: Long = System.currentTimeMillis()): Boolean {
val since = context.getSharedPreferences(STORE, Context.MODE_PRIVATE)
.getLong(KEY_ACTIVE_SINCE, 0L)
return isFresh(since, nowMs)
}
internal fun isFresh(sinceMs: Long, nowMs: Long): Boolean =
sinceMs > 0L && nowMs - sinceMs in 0..STALE_AFTER_MS
}
object AssistantAppSessionState {
private val _active = MutableStateFlow(false)
val active: StateFlow<Boolean> = _active.asStateFlow()
@Volatile private var voiceStarted = false
internal fun setActive(active: Boolean) {
if (active && !_active.value) voiceStarted = false
if (!active) voiceStarted = false
_active.value = active
}
fun markVoiceStarted() {
voiceStarted = true
}
fun hasVoiceStarted(): Boolean = voiceStarted
}
object AssistantVoiceCommandCoordinator {
private val _cancelRequest = MutableStateFlow<String?>(null)
val cancelRequest: StateFlow<String?> = _cancelRequest.asStateFlow()
fun requestCancel() {
_cancelRequest.value = UUID.randomUUID().toString()
}
fun consume(id: String): Boolean {
if (_cancelRequest.value != id) return false
_cancelRequest.value = null
return true
}
}
@@ -0,0 +1,26 @@
package com.hermesandroid.relay.assistant
import android.content.Intent
import android.speech.RecognitionService
import android.speech.SpeechRecognizer
/**
* Platform-required recognition component for the Hermes voice interactor.
*
* Assistant sessions deliberately use the existing Hermes transcription
* pipeline so wake detection, session capture, and active voice never compete
* for the microphone. Direct SpeechRecognizer clients are therefore rejected
* instead of opening a second recorder.
*/
class HermesRecognitionService : RecognitionService() {
override fun onStartListening(
recognizerIntent: Intent,
listener: Callback,
) {
listener.error(SpeechRecognizer.ERROR_CLIENT)
}
override fun onStopListening(listener: Callback) = Unit
override fun onCancel(listener: Callback) = Unit
}
@@ -0,0 +1,299 @@
package com.hermesandroid.relay.assistant
import android.Manifest
import android.annotation.SuppressLint
import android.content.pm.PackageManager
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.service.voice.VoiceInteractionService
import android.util.Log
import androidx.core.content.ContextCompat
import com.hermesandroid.relay.wake.MicrophoneLease
import com.hermesandroid.relay.wake.MicrophoneOwner
import com.hermesandroid.relay.wake.MicrophoneOwnershipCoordinator
import com.hermesandroid.relay.wake.SherpaWakeWordDetector
import com.hermesandroid.relay.wake.WakeWordModelInstaller
import com.hermesandroid.relay.wake.WakeWordPreferences
import com.hermesandroid.relay.wake.WakeWordPreferencesRepository
import java.util.concurrent.atomic.AtomicBoolean
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.collectLatest
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
enum class AssistantWakeRuntimeState {
Stopped,
Starting,
Listening,
PausedForVoice,
AwaitingSession,
Error,
}
/**
* Opt-in Android Digital Assistant service. Android keeps the selected service
* available in the background; all pre-activation audio is evaluated locally.
*/
class HermesVoiceInteractionService : VoiceInteractionService() {
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val mainHandler = Handler(Looper.getMainLooper())
private val stopRequested = AtomicBoolean(false)
private val resourceLock = Any()
private var preferencesJob: Job? = null
private var recognitionJob: Job? = null
private var recorder: AudioRecord? = null
private var detector: SherpaWakeWordDetector? = null
private var microphoneLease: MicrophoneLease? = null
@Volatile private var latestPreferences = WakeWordPreferences()
@Volatile private var voiceSessionActive = false
override fun onCreate() {
super.onCreate()
runningInstance = this
}
override fun onReady() {
super.onReady()
if (runningInstance !== this) return
voiceSessionActive = AssistantSessionPersistence.isActive(this)
preferencesJob?.cancel()
preferencesJob = scope.launch {
WakeWordPreferencesRepository(applicationContext).flow.collectLatest { prefs ->
latestPreferences = prefs
if (prefs.assistantEnabled && !voiceSessionActive) {
restartRecognition(prefs)
} else {
stopRecognition()
setRuntimeState(
if (voiceSessionActive) {
AssistantWakeRuntimeState.PausedForVoice
} else {
AssistantWakeRuntimeState.Stopped
}
)
}
}
}
}
override fun onLaunchVoiceAssistFromKeyguard() {
val activationId = java.util.UUID.randomUUID().toString()
showAssistantSession(
fromKeyguard = true,
activationId = activationId,
)
}
override fun onShutdown() {
stopRecognition()
preferencesJob?.cancel()
setRuntimeState(AssistantWakeRuntimeState.Stopped)
super.onShutdown()
}
override fun onDestroy() {
stopRecognition()
preferencesJob?.cancel()
if (runningInstance === this) runningInstance = null
scope.cancel()
super.onDestroy()
}
private suspend fun restartRecognition(preferences: WakeWordPreferences) {
val previous = recognitionJob
stopRecognition()
previous?.join()
if (!voiceSessionActive && preferences.assistantEnabled) {
startRecognition(preferences)
}
}
@SuppressLint("MissingPermission")
private fun startRecognition(preferences: WakeWordPreferences) {
if (voiceSessionActive || recognitionJob?.isActive == true) return
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) !=
PackageManager.PERMISSION_GRANTED
) {
setRuntimeState(AssistantWakeRuntimeState.Error)
return
}
val files = WakeWordModelInstaller(this).installedFiles()
if (files == null) {
setRuntimeState(AssistantWakeRuntimeState.Error)
return
}
val lease = MicrophoneOwnershipCoordinator.tryAcquire(MicrophoneOwner.WakeWord)
if (lease == null) {
setRuntimeState(AssistantWakeRuntimeState.PausedForVoice)
scheduleRetry()
return
}
microphoneLease = lease
stopRequested.set(false)
setRuntimeState(AssistantWakeRuntimeState.Starting)
recognitionJob = scope.launch {
var detected = false
var unattachedDetector: SherpaWakeWordDetector? = null
try {
val createdDetector = SherpaWakeWordDetector(
files,
preferences.sensitivity,
preferences.confirmationFrames,
)
unattachedDetector = createdDetector
val minBuffer = AudioRecord.getMinBufferSize(
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
).coerceAtLeast(SAMPLE_RATE / 5 * 2)
val createdRecorder = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)
.setAudioFormat(
AudioFormat.Builder()
.setSampleRate(SAMPLE_RATE)
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
.setBufferSizeInBytes(minBuffer * 2)
.build()
if (createdRecorder.state != AudioRecord.STATE_INITIALIZED) {
createdRecorder.release()
error("Assistant wake microphone failed to initialize")
}
synchronized(resourceLock) {
if (stopRequested.get()) {
createdRecorder.release()
return@launch
}
recorder = createdRecorder
detector = createdDetector
unattachedDetector = null
}
createdRecorder.startRecording()
setRuntimeState(AssistantWakeRuntimeState.Listening)
val samples = ShortArray(FRAME_SAMPLES)
while (!stopRequested.get()) {
val count = createdRecorder.read(samples, 0, samples.size)
if (count < 0) error("Assistant wake microphone read failed: $count")
if (count > 0 && createdDetector.accept(samples, count)) {
detected = true
break
}
}
} catch (t: Throwable) {
if (!stopRequested.get()) {
Log.w(TAG, "Assistant wake listening failed", t)
setRuntimeState(AssistantWakeRuntimeState.Error)
}
} finally {
runCatching { unattachedDetector?.close() }
releaseResources()
recognitionJob = null
}
if (detected && !stopRequested.get()) {
setRuntimeState(AssistantWakeRuntimeState.AwaitingSession)
val keyguard = getSystemService(android.app.KeyguardManager::class.java)
mainHandler.post {
showAssistantSession(fromKeyguard = keyguard?.isKeyguardLocked == true)
}
}
}
}
private fun showAssistantSession(fromKeyguard: Boolean, activationId: String? = null) {
voiceSessionActive = true
stopRecognition()
setRuntimeState(AssistantWakeRuntimeState.AwaitingSession)
showSession(
Bundle().apply {
putBoolean(EXTRA_FROM_KEYGUARD, fromKeyguard)
activationId?.let { putString(AssistantSessionProtocol.EXTRA_ACTIVATION_ID, it) }
putBoolean(
AssistantSessionProtocol.EXTRA_START_NEW_SESSION,
latestPreferences.startNewSession,
)
},
0,
)
}
private fun setVoiceSessionActiveInternal(active: Boolean) {
voiceSessionActive = active
if (active) {
stopRecognition()
setRuntimeState(AssistantWakeRuntimeState.PausedForVoice)
} else if (latestPreferences.assistantEnabled) {
scheduleRetry()
} else {
setRuntimeState(AssistantWakeRuntimeState.Stopped)
}
}
private fun scheduleRetry() {
if (recognitionJob?.isActive == true || voiceSessionActive) return
recognitionJob = scope.launch {
delay(RETRY_DELAY_MS)
recognitionJob = null
if (!voiceSessionActive && latestPreferences.assistantEnabled) {
startRecognition(latestPreferences)
}
}
}
private fun stopRecognition() {
stopRequested.set(true)
synchronized(resourceLock) {
runCatching { recorder?.stop() }
runCatching { recorder?.release() }
recorder = null
microphoneLease?.let(MicrophoneOwnershipCoordinator::release)
microphoneLease = null
}
recognitionJob?.cancel()
}
private fun releaseResources() {
synchronized(resourceLock) {
runCatching { recorder?.stop() }
runCatching { recorder?.release() }
recorder = null
runCatching { detector?.close() }
detector = null
microphoneLease?.let(MicrophoneOwnershipCoordinator::release)
microphoneLease = null
}
}
private fun setRuntimeState(state: AssistantWakeRuntimeState) {
_runtimeState.value = state
}
companion object {
private const val TAG = "HermesAssistant"
private const val SAMPLE_RATE = 16_000
private const val FRAME_SAMPLES = 1_600
private const val RETRY_DELAY_MS = 500L
const val EXTRA_FROM_KEYGUARD = "from_keyguard"
private val _runtimeState = kotlinx.coroutines.flow.MutableStateFlow(
AssistantWakeRuntimeState.Stopped
)
val runtimeState = _runtimeState.asStateFlow()
@Volatile private var runningInstance: HermesVoiceInteractionService? = null
fun setVoiceSessionActive(active: Boolean) {
runningInstance?.setVoiceSessionActiveInternal(active)
}
}
}
@@ -0,0 +1,629 @@
package com.hermesandroid.relay.assistant
import android.graphics.drawable.ColorDrawable
import android.os.Bundle
import android.service.voice.VoiceInteractionSession
import android.service.voice.VoiceInteractionSessionService
import android.view.View
import android.view.WindowManager
import androidx.compose.animation.animateContentSize
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.navigationBarsPadding
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.AutoAwesome
import androidx.compose.material.icons.filled.ExpandLess
import androidx.compose.material.icons.filled.ExpandMore
import androidx.compose.material.icons.filled.GraphicEq
import androidx.compose.material.icons.filled.Person
import androidx.compose.material.icons.filled.Stop
import androidx.compose.material3.Button
import androidx.compose.material3.ButtonDefaults
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.StrokeCap
import androidx.compose.ui.layout.boundsInWindow
import androidx.compose.ui.layout.onGloballyPositioned
import androidx.compose.ui.platform.ComposeView
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.LifecycleRegistry
import androidx.lifecycle.ViewModelStore
import androidx.lifecycle.ViewModelStoreOwner
import androidx.lifecycle.setViewTreeLifecycleOwner
import androidx.lifecycle.setViewTreeViewModelStoreOwner
import androidx.savedstate.SavedStateRegistry
import androidx.savedstate.SavedStateRegistryController
import androidx.savedstate.SavedStateRegistryOwner
import androidx.savedstate.setViewTreeSavedStateRegistryOwner
import com.hermesandroid.relay.R
import com.hermesandroid.relay.ui.theme.HermesRelayTheme
import java.util.UUID
import kotlin.math.max
import kotlin.math.roundToInt
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
class HermesVoiceInteractionSessionService : VoiceInteractionSessionService() {
override fun onNewSession(args: Bundle?): VoiceInteractionSession =
HermesVoiceInteractionSession(this)
}
internal enum class AssistantSessionPresentation {
Inactive,
Overlay,
FullVoice,
}
internal fun shouldCancelVoiceWhenSessionUiEnds(
presentation: AssistantSessionPresentation,
): Boolean = presentation == AssistantSessionPresentation.Overlay
private class HermesVoiceInteractionSession(
private val service: HermesVoiceInteractionSessionService,
) : VoiceInteractionSession(service) {
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)
private val viewOwner = AssistantSessionViewOwner().also { it.start() }
private var presentation = AssistantSessionPresentation.Inactive
private val assistantSurfaceBounds = android.graphics.Rect()
private var surfaceExpanded by mutableStateOf(false)
init {
scope.launch {
AssistantSessionState.snapshot.collect { snapshot ->
if (presentation != AssistantSessionPresentation.Inactive &&
snapshot.phase == AssistantSessionPhase.Closed
) {
finishSession(cancelVoice = false)
}
}
}
}
override fun onCreate() {
super.onCreate()
window.window?.apply {
setBackgroundDrawable(ColorDrawable(android.graphics.Color.TRANSPARENT))
clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND)
setDimAmount(0f)
}
}
override fun onCreateContentView(): View = ComposeView(service).apply {
setBackgroundColor(android.graphics.Color.TRANSPARENT)
setViewTreeLifecycleOwner(viewOwner)
setViewTreeViewModelStoreOwner(viewOwner)
setViewTreeSavedStateRegistryOwner(viewOwner)
setContent {
HermesRelayTheme {
AssistantSessionSurface(
expanded = surfaceExpanded,
onExpandedChange = { surfaceExpanded = it },
onCancel = { finishSession(cancelVoice = true) },
onRetry = { launchVoice(startNewSession = true) },
onOpenFullVoice = {
if (presentation == AssistantSessionPresentation.Overlay) {
openFullVoice()
}
},
onSurfaceBoundsChanged = { bounds ->
if (assistantSurfaceBounds != bounds) {
assistantSurfaceBounds.set(bounds)
window.window?.decorView?.requestLayout()
}
},
)
}
}
}
override fun onShow(args: Bundle?, showFlags: Int) {
super.onShow(args, showFlags)
if (args?.getBoolean(HermesVoiceInteractionService.EXTRA_FROM_KEYGUARD, false) == true) {
window.window?.addFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
}
setUiEnabled(true)
val startsNewLifecycle = presentation == AssistantSessionPresentation.Inactive
presentation = AssistantSessionPresentation.Overlay
if (!startsNewLifecycle) return
surfaceExpanded = false
AssistantSessionState.reset()
launchVoice(
activationId = args?.getString(AssistantSessionProtocol.EXTRA_ACTIVATION_ID)
?: UUID.randomUUID().toString(),
startNewSession = args?.getBoolean(
AssistantSessionProtocol.EXTRA_START_NEW_SESSION,
true,
) ?: true,
)
}
override fun onComputeInsets(outInsets: Insets) {
super.onComputeInsets(outInsets)
outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_REGION
outInsets.touchableRegion.set(assistantSurfaceBounds)
}
override fun onBackPressed() {
if (presentation == AssistantSessionPresentation.Overlay && surfaceExpanded) {
surfaceExpanded = false
return
}
super.onBackPressed()
}
override fun onHide() {
if (shouldCancelVoiceWhenSessionUiEnds(presentation)) {
finishSession(cancelVoice = true)
}
super.onHide()
}
override fun onDestroy() {
if (shouldCancelVoiceWhenSessionUiEnds(presentation)) {
AssistantSessionProtocol.finish(service, cancelVoice = true)
}
presentation = AssistantSessionPresentation.Inactive
viewOwner.stop()
scope.cancel()
super.onDestroy()
}
private fun launchVoice(
activationId: String = UUID.randomUUID().toString(),
startNewSession: Boolean,
) {
runCatching {
AssistantSessionProtocol.activate(
service,
activationId = activationId,
startNewSession = startNewSession,
)
}.onFailure {
AssistantSessionState.update(
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open the voice session.",
)
)
}
}
private fun openFullVoice() {
runCatching {
startVoiceActivity(AssistantSessionProtocol.fullVoiceIntent(service))
presentation = AssistantSessionPresentation.FullVoice
setUiEnabled(false)
}.onFailure {
AssistantSessionState.update(
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open full voice.",
)
)
}
}
private fun finishSession(cancelVoice: Boolean) {
if (presentation == AssistantSessionPresentation.Inactive) return
presentation = AssistantSessionPresentation.Inactive
AssistantSessionProtocol.finish(service, cancelVoice)
finish()
}
}
private class AssistantSessionViewOwner :
LifecycleOwner,
ViewModelStoreOwner,
SavedStateRegistryOwner {
private val lifecycleRegistry = LifecycleRegistry(this)
private val store = ViewModelStore()
private val savedStateController = SavedStateRegistryController.create(this)
override val lifecycle: Lifecycle get() = lifecycleRegistry
override val viewModelStore: ViewModelStore get() = store
override val savedStateRegistry: SavedStateRegistry
get() = savedStateController.savedStateRegistry
fun start() {
savedStateController.performRestore(null)
lifecycleRegistry.currentState = Lifecycle.State.CREATED
lifecycleRegistry.currentState = Lifecycle.State.RESUMED
}
fun stop() {
lifecycleRegistry.currentState = Lifecycle.State.DESTROYED
store.clear()
}
}
@Composable
private fun AssistantSessionSurface(
expanded: Boolean,
onExpandedChange: (Boolean) -> Unit,
onCancel: () -> Unit,
onRetry: () -> Unit,
onOpenFullVoice: () -> Unit,
onSurfaceBoundsChanged: (android.graphics.Rect) -> Unit,
) {
val snapshot by AssistantSessionState.snapshot.collectAsState()
val status = assistantStatus(snapshot.phase)
Box(
modifier = Modifier
.fillMaxSize()
.padding(horizontal = 12.dp, vertical = 12.dp)
.navigationBarsPadding(),
contentAlignment = Alignment.BottomCenter,
) {
Surface(
modifier = Modifier
.fillMaxWidth()
.animateContentSize()
.onGloballyPositioned { coordinates ->
val bounds = coordinates.boundsInWindow()
onSurfaceBoundsChanged(
android.graphics.Rect(
bounds.left.roundToInt(),
bounds.top.roundToInt(),
bounds.right.roundToInt(),
bounds.bottom.roundToInt(),
)
)
},
shape = RoundedCornerShape(if (expanded) 30.dp else 28.dp),
color = MaterialTheme.colorScheme.surfaceContainerHigh.copy(alpha = 0.98f),
contentColor = MaterialTheme.colorScheme.onSurface,
tonalElevation = 10.dp,
shadowElevation = 12.dp,
) {
if (expanded) {
ExpandedAssistantSurface(
snapshot = snapshot,
status = status,
onCollapse = { onExpandedChange(false) },
onCancel = onCancel,
onRetry = onRetry,
onOpenFullVoice = onOpenFullVoice,
)
} else {
CompactAssistantSurface(
snapshot = snapshot,
status = status,
onExpand = { onExpandedChange(true) },
onCancel = onCancel,
)
}
}
}
}
@Composable
private fun CompactAssistantSurface(
snapshot: AssistantSessionSnapshot,
status: String,
onExpand: () -> Unit,
onCancel: () -> Unit,
) {
Row(
modifier = Modifier.padding(horizontal = 14.dp, vertical = 12.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(12.dp),
) {
AssistantOrb(snapshot.phase)
Column(modifier = Modifier.weight(1f)) {
Text(
text = status,
style = MaterialTheme.typography.labelLarge,
color = MaterialTheme.colorScheme.primary,
fontWeight = FontWeight.SemiBold,
)
Text(
text = compactAssistantText(snapshot),
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurface,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
IconButton(
onClick = onExpand,
modifier = Modifier.size(40.dp),
) {
Icon(
imageVector = Icons.Filled.ExpandLess,
contentDescription = stringResource(R.string.assistant_session_expand),
)
}
AssistantStopButton(onClick = onCancel, compact = true)
}
}
@Composable
private fun ExpandedAssistantSurface(
snapshot: AssistantSessionSnapshot,
status: String,
onCollapse: () -> Unit,
onCancel: () -> Unit,
onRetry: () -> Unit,
onOpenFullVoice: () -> Unit,
) {
Column(
modifier = Modifier.padding(horizontal = 20.dp, vertical = 12.dp),
verticalArrangement = Arrangement.spacedBy(12.dp),
) {
Box(
modifier = Modifier
.width(38.dp)
.height(4.dp)
.clip(CircleShape)
.background(MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.5f))
.align(Alignment.CenterHorizontally),
)
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
AssistantOrb(snapshot.phase, size = 38)
Column(modifier = Modifier.weight(1f)) {
Text(
text = stringResource(R.string.app_name),
style = MaterialTheme.typography.titleMedium,
fontWeight = FontWeight.SemiBold,
)
Text(
text = status,
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.primary,
)
}
IconButton(onClick = onCollapse) {
Icon(
imageVector = Icons.Filled.ExpandMore,
contentDescription = stringResource(R.string.assistant_session_collapse),
)
}
}
AssistantWaveform(snapshot.phase)
snapshot.transcript?.takeIf { it.isNotBlank() }?.let { transcript ->
AssistantTextRow(
icon = Icons.Filled.Person,
text = transcript,
color = MaterialTheme.colorScheme.primary,
)
}
snapshot.response.takeIf { it.isNotBlank() }?.let { response ->
AssistantTextRow(
icon = Icons.Filled.AutoAwesome,
text = response,
color = MaterialTheme.colorScheme.onSurface,
)
}
snapshot.error?.let { error ->
Text(
text = error,
color = MaterialTheme.colorScheme.error,
style = MaterialTheme.typography.bodyMedium,
)
}
if (snapshot.phase == AssistantSessionPhase.Transcribing ||
snapshot.phase == AssistantSessionPhase.Thinking
) {
LinearProgressIndicator(modifier = Modifier.fillMaxWidth())
}
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(10.dp),
verticalAlignment = Alignment.CenterVertically,
) {
AssistantStopButton(onClick = onCancel, compact = false)
Spacer(Modifier.weight(1f))
if (snapshot.phase == AssistantSessionPhase.Error) {
TextButton(onClick = onRetry) {
Text(stringResource(R.string.assistant_session_retry))
}
}
OutlinedButton(onClick = onOpenFullVoice) {
Text(stringResource(R.string.assistant_session_open_full_voice))
}
}
}
}
@Composable
private fun AssistantOrb(
phase: AssistantSessionPhase,
size: Int = 52,
) {
val active = phase == AssistantSessionPhase.Listening ||
phase == AssistantSessionPhase.Transcribing ||
phase == AssistantSessionPhase.Thinking ||
phase == AssistantSessionPhase.Speaking
Box(
modifier = Modifier
.size(size.dp)
.clip(CircleShape)
.background(
if (active) {
MaterialTheme.colorScheme.primaryContainer
} else {
MaterialTheme.colorScheme.surfaceVariant
}
),
contentAlignment = Alignment.Center,
) {
Icon(
imageVector = Icons.Filled.GraphicEq,
contentDescription = null,
tint = if (active) {
MaterialTheme.colorScheme.onPrimaryContainer
} else {
MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = Modifier.size((size * 0.5f).dp),
)
}
}
@Composable
private fun AssistantWaveform(phase: AssistantSessionPhase) {
val active = phase == AssistantSessionPhase.Listening ||
phase == AssistantSessionPhase.Speaking
val primary = if (active) {
MaterialTheme.colorScheme.primary
} else {
MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.55f)
}
Canvas(
modifier = Modifier
.fillMaxWidth()
.height(28.dp),
) {
val centerY = size.height / 2f
val bars = 33
val spacing = size.width / bars
repeat(bars) { index ->
val distance = kotlin.math.abs(index - bars / 2f) / (bars / 2f)
val envelope = max(0.18f, 1f - distance)
val pattern = 0.45f + ((index * 17) % 11) / 20f
val halfHeight = size.height * 0.46f * envelope * pattern
val x = spacing * (index + 0.5f)
drawLine(
color = primary,
start = Offset(x, centerY - halfHeight),
end = Offset(x, centerY + halfHeight),
strokeWidth = max(2f, spacing * 0.28f),
cap = StrokeCap.Round,
)
}
}
}
@Composable
private fun AssistantTextRow(
icon: androidx.compose.ui.graphics.vector.ImageVector,
text: String,
color: Color,
) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(12.dp),
verticalAlignment = Alignment.Top,
) {
Icon(
imageVector = icon,
contentDescription = null,
tint = color,
modifier = Modifier.size(20.dp),
)
Text(
text = text,
style = MaterialTheme.typography.bodyLarge,
color = color,
maxLines = 4,
overflow = TextOverflow.Ellipsis,
)
}
}
@Composable
private fun AssistantStopButton(
onClick: () -> Unit,
compact: Boolean,
) {
if (compact) {
IconButton(
onClick = onClick,
modifier = Modifier
.size(44.dp)
.clip(CircleShape)
.background(MaterialTheme.colorScheme.errorContainer),
) {
Icon(
imageVector = Icons.Filled.Stop,
contentDescription = stringResource(R.string.assistant_session_cancel),
tint = MaterialTheme.colorScheme.error,
)
}
} else {
Button(
onClick = onClick,
colors = ButtonDefaults.buttonColors(
containerColor = MaterialTheme.colorScheme.errorContainer,
contentColor = MaterialTheme.colorScheme.error,
),
) {
Icon(
imageVector = Icons.Filled.Stop,
contentDescription = null,
modifier = Modifier.size(18.dp),
)
Spacer(Modifier.width(8.dp))
Text(stringResource(R.string.assistant_session_stop))
}
}
}
@Composable
private fun assistantStatus(phase: AssistantSessionPhase): String = when (phase) {
AssistantSessionPhase.Launching -> stringResource(R.string.assistant_session_launching)
AssistantSessionPhase.Listening -> stringResource(R.string.assistant_session_listening)
AssistantSessionPhase.Transcribing -> stringResource(R.string.assistant_session_transcribing)
AssistantSessionPhase.Thinking -> stringResource(R.string.assistant_session_thinking)
AssistantSessionPhase.Speaking -> stringResource(R.string.assistant_session_speaking)
AssistantSessionPhase.Idle -> stringResource(R.string.assistant_session_ready)
AssistantSessionPhase.Error -> stringResource(R.string.assistant_session_error)
AssistantSessionPhase.Closed -> stringResource(R.string.assistant_session_closing)
}
@Composable
private fun compactAssistantText(snapshot: AssistantSessionSnapshot): String =
snapshot.transcript?.takeIf { it.isNotBlank() }
?: snapshot.response.takeIf { it.isNotBlank() }
?: snapshot.error?.takeIf { it.isNotBlank() }
?: assistantStatus(snapshot.phase)
@@ -8,11 +8,16 @@ import android.media.MediaRecorder
import android.media.audiofx.AcousticEchoCanceler
import android.media.audiofx.NoiseSuppressor
import android.util.Log
import com.hermesandroid.relay.wake.MicrophoneLease
import com.hermesandroid.relay.wake.MicrophoneOwner
import com.hermesandroid.relay.wake.MicrophoneOwnershipCoordinator
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.MutableStateFlow
@@ -22,23 +27,26 @@ import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.coroutines.yield
import kotlin.math.max
/**
* Duplex audio capture for voice barge-in (plan unit B3).
*
* While TTS is playing, this listener continuously pulls 32 ms / 512-sample
* PCM frames off the microphone and feeds them to [VadEngine]. It emits two
* SharedFlows that B4 will wire into the voice state machine:
* During response generation and playback, this listener continuously pulls
* 32 ms / 512-sample PCM frames off the microphone and feeds them to
* [VadEngine]. One instance owns the full active turn. It emits two
* SharedFlows wired into the voice state machine:
*
* - [maybeSpeech] fires on the **first** positive raw-VAD frame — before the
* second-layer debouncer latches. B4 uses this to softly [VoicePlayer.duck]
* the TTS so the user's voice has acoustic headroom while we decide whether
* to cut off.
*
* - [bargeInDetected] fires when [VadEngine] confirms speech post-hysteresis.
* B4 uses this to call `interruptSpeaking()` and flip state to Listening.
* - [bargeInDetected] fires when [VadEngine] confirms speech post-hysteresis
* and the calibrated RMS majority gate accepts it. The owner uses this to
* interrupt generation/playback and flip state to Listening.
*
* ### Acoustic echo cancellation
*
@@ -140,8 +148,43 @@ class BargeInListener internal constructor(
private val frameBuffer: ShortArray = ShortArray(VadEngine.FRAME_SIZE_SAMPLES)
@Volatile private var readerJob: Job? = null
@Volatile private var microphoneLease: MicrophoneLease? = null
@Volatile private var aec: AcousticEchoCanceler? = null
@Volatile private var noiseSuppressor: NoiseSuppressor? = null
private val rmsGate = RmsBargeInGate()
@Volatile private var playbackGraceMs: Long = RmsBargeInGate.DEFAULT_PLAYBACK_GRACE_MS
@Volatile private var playbackActiveProvider: (() -> Boolean)? = null
@Volatile private var diagnosticsEnabled: Boolean = false
private var wasCalibrating: Boolean = false
/** Apply the user-facing barge-in sensitivity to the quiet-room RMS gate. */
fun setThresholdMultiplier(multiplier: Float) {
rmsGate.thresholdMultiplier = multiplier
}
fun setDiagnosticsEnabled(enabled: Boolean) {
diagnosticsEnabled = enabled
}
/** Supplies the renderer's current playback phase for upstream-style gaps. */
fun setPlaybackActiveProvider(provider: () -> Boolean) {
playbackActiveProvider = provider
}
/**
* Freeze quiet-room calibration and begin the playback-only grace window.
* Idempotent so every renderer may call it at its first audible chunk.
*/
fun markPlaybackStarted(
nowMs: Long = System.currentTimeMillis(),
graceMs: Long = RmsBargeInGate.DEFAULT_PLAYBACK_GRACE_MS,
) {
playbackGraceMs = graceMs.coerceAtLeast(0L)
rmsGate.markPlaybackStarted(nowMs)
if (diagnosticsEnabled) {
Log.d(TAG, "voice-vad playback started; grace=${playbackGraceMs}ms")
}
}
/**
* Allocate the audio pipeline and begin reading frames into [vadEngine].
@@ -163,6 +206,12 @@ class BargeInListener internal constructor(
return
}
val lease = MicrophoneOwnershipCoordinator.tryAcquire(MicrophoneOwner.BargeIn)
if (lease == null) {
Log.i(TAG, "Barge-in listener inactive — microphone is owned by another voice surface")
return
}
microphoneLease = lease
if (!audioSource.initialize()) {
Log.w(
TAG,
@@ -170,11 +219,16 @@ class BargeInListener internal constructor(
"(missing RECORD_AUDIO permission or mic busy) — listener inactive",
)
_aecAttached.value = false
MicrophoneOwnershipCoordinator.release(lease)
microphoneLease = null
return
}
_aecAttached.value = false
rmsGate.reset()
wasCalibrating = true
readerJob = scope.launch(readerDispatcher) {
var effectsJob: Job? = null
try {
try {
audioSource.start()
@@ -185,7 +239,10 @@ class BargeInListener internal constructor(
return@launch
}
Log.i(TAG, "Barge-in AudioRecord reader started")
maybeAttachEffects()
// Do not block generation-phase listening while waiting for an
// AudioTrack session that does not exist until playback. The
// effects attach races harmlessly beside the reader.
effectsJob = launch { maybeAttachEffects() }
while (isActive) {
val read = try {
@@ -222,10 +279,41 @@ class BargeInListener internal constructor(
Log.w(TAG, "VadEngine.analyze failed; stopping reader: ${t.message}")
break
}
if (result.probability > 0f) {
val gated = rmsGate.observe(
frame = frameBuffer,
rawSpeech = result.probability > 0f,
nowMs = System.currentTimeMillis(),
playbackGraceMs = playbackGraceMs,
confirmedSpeech = result.isSpeech,
playbackActiveOverride = playbackActiveProvider?.invoke(),
)
if (diagnosticsEnabled) {
if (wasCalibrating && !gated.calibrating) {
Log.d(
TAG,
"voice-vad calibrated quiet floor=${gated.floor.toInt()} " +
"mult=${rmsGate.thresholdMultiplier}",
)
}
wasCalibrating = gated.calibrating
if (
gated.detected || gated.playbackGrace ||
gated.rms >= gated.threshold * 0.5f
) {
Log.d(
TAG,
"voice-vad rms=${gated.rms.toInt()} floor=${gated.floor.toInt()} " +
"trigger=${gated.threshold.toInt()} raw=${result.probability > 0f} " +
"confirmed=${result.isSpeech} detected=${gated.detected} " +
"grace=${gated.playbackGrace} " +
"phase=${if (gated.playback) "playback" else "generation"}",
)
}
}
if (gated.maybeSpeech) {
_maybeSpeech.tryEmit(Unit)
}
if (result.isSpeech) {
if (gated.detected) {
_bargeInDetected.tryEmit(Unit)
}
// Give the dispatcher a chance to observe cancellation
@@ -237,11 +325,19 @@ class BargeInListener internal constructor(
yield()
}
} finally {
// The reader reaches this block with its Job cancelled.
// Teardown still has to wait for the sibling AEC poll before
// releasing the AudioRecord and microphone lease.
withContext(NonCancellable) {
effectsJob?.cancelAndJoin()
}
// Release effects + AudioRecord in the reverse of attach order
// so the AudioSessionId is still valid when AEC teardown runs.
releaseEffects()
runCatching { audioSource.stop() }
runCatching { audioSource.release() }
microphoneLease?.let(MicrophoneOwnershipCoordinator::release)
microphoneLease = null
_aecAttached.value = false
}
}
@@ -258,8 +354,16 @@ class BargeInListener internal constructor(
if (job?.isActive == true) {
Log.i(TAG, "Stopping barge-in AudioRecord reader")
}
// AudioRecord.read() may be blocked in native code, so stop the source
// before cancellation to make the reader observe shutdown promptly.
runCatching { audioSource.stop() }
job?.cancel()
readerJob = null
if (job == null) {
runCatching { audioSource.release() }
microphoneLease?.let(MicrophoneOwnershipCoordinator::release)
microphoneLease = null
}
return job
}
@@ -4,6 +4,8 @@ import android.annotation.SuppressLint
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import com.hermesandroid.relay.wake.MicrophoneOwner
import com.hermesandroid.relay.wake.MicrophoneOwnershipCoordinator
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream
@@ -40,24 +42,37 @@ class RealtimePcmRecorder(
maxDurationMs: Long = 15_000,
onLevel: ((Float) -> Unit)? = null,
): ByteArray = withContext(Dispatchers.IO) {
val minBuffer = AudioRecord.getMinBufferSize(
sampleRate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
).coerceAtLeast(sampleRate / 10 * 2)
val microphoneLease =
MicrophoneOwnershipCoordinator.tryAcquire(MicrophoneOwner.RealtimeDiagnostics)
?: error("Microphone is in use by another voice feature")
val minBuffer = try {
AudioRecord.getMinBufferSize(
sampleRate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
).coerceAtLeast(sampleRate / 10 * 2)
} catch (t: Throwable) {
MicrophoneOwnershipCoordinator.release(microphoneLease)
throw t
}
val maxBytes = ((sampleRate * maxDurationMs) / 1000L * 2L).toInt()
val recorder = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(sampleRate)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
.setBufferSizeInBytes(minBuffer)
.build()
val recorder = try {
AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(sampleRate)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
.setBufferSizeInBytes(minBuffer)
.build()
} catch (t: Throwable) {
MicrophoneOwnershipCoordinator.release(microphoneLease)
throw t
}
val out = ByteArrayOutputStream(minBuffer * 4)
val buffer = ByteArray(minBuffer)
@@ -77,32 +92,46 @@ class RealtimePcmRecorder(
capturing = false
try { recorder.stop() } catch (_: Exception) { }
recorder.release()
MicrophoneOwnershipCoordinator.release(microphoneLease)
}
out.toByteArray()
}
@SuppressLint("MissingPermission")
suspend fun capture(durationMs: Long = 800): ByteArray = withContext(Dispatchers.IO) {
val minBuffer = AudioRecord.getMinBufferSize(
sampleRate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
).coerceAtLeast(sampleRate / 10 * 2)
val microphoneLease =
MicrophoneOwnershipCoordinator.tryAcquire(MicrophoneOwner.RealtimeDiagnostics)
?: error("Microphone is in use by another voice feature")
val minBuffer = try {
AudioRecord.getMinBufferSize(
sampleRate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
).coerceAtLeast(sampleRate / 10 * 2)
} catch (t: Throwable) {
MicrophoneOwnershipCoordinator.release(microphoneLease)
throw t
}
val targetBytes = ((sampleRate * durationMs) / 1000L * 2L)
.toInt()
.coerceAtLeast(minBuffer)
val recorder = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(sampleRate)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
.setBufferSizeInBytes(minBuffer)
.build()
val recorder = try {
AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(sampleRate)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
.setBufferSizeInBytes(minBuffer)
.build()
} catch (t: Throwable) {
MicrophoneOwnershipCoordinator.release(microphoneLease)
throw t
}
val out = ByteArrayOutputStream(targetBytes)
val buffer = ByteArray(minBuffer)
@@ -123,6 +152,7 @@ class RealtimePcmRecorder(
} finally {
try { recorder.stop() } catch (_: Exception) { }
recorder.release()
MicrophoneOwnershipCoordinator.release(microphoneLease)
}
out.toByteArray()
}
@@ -0,0 +1,216 @@
package com.hermesandroid.relay.audio
import kotlin.math.ceil
import kotlin.math.roundToInt
import kotlin.math.sqrt
/**
* Turn-scoped RMS gate layered in front of the model VAD.
*
* The first quiet frames establish a room floor before playback. That floor is
* frozen as soon as playback begins so speaker output can never teach the gate
* to ignore the user. Detection uses a majority window rather than requiring
* perfectly consecutive frames, which tolerates short consonant/syllable dips.
*/
internal class RmsBargeInGate(
private val calibrationFrames: Int = DEFAULT_CALIBRATION_FRAMES,
private val decisionWindowFrames: Int = DEFAULT_DECISION_WINDOW_FRAMES,
private val requiredWindowRatio: Float = DEFAULT_REQUIRED_WINDOW_RATIO,
) {
private val ambient = ArrayDeque<Float>(MAX_AMBIENT_FRAMES)
private val decisions = ArrayDeque<Boolean>(decisionWindowFrames)
private var quietFloor: Float = DEFAULT_QUIET_FLOOR_RMS
private var calibrated = false
private var playbackActive = false
private var playbackStartedAtMs: Long? = null
private var playbackStoppedAtMs: Long? = null
var thresholdMultiplier: Float = DEFAULT_THRESHOLD_MULTIPLIER
set(value) {
field = value.coerceIn(MIN_THRESHOLD_MULTIPLIER, MAX_THRESHOLD_MULTIPLIER)
}
fun reset() {
ambient.clear()
decisions.clear()
quietFloor = DEFAULT_QUIET_FLOOR_RMS
calibrated = false
playbackActive = false
playbackStartedAtMs = null
playbackStoppedAtMs = null
}
fun markPlaybackStarted(nowMs: Long) {
updatePlaybackPhase(active = true, nowMs = nowMs)
}
fun observe(
frame: ShortArray,
rawSpeech: Boolean,
nowMs: Long,
playbackGraceMs: Long,
confirmedSpeech: Boolean = rawSpeech,
playbackActiveOverride: Boolean? = null,
): RmsGateResult {
val rms = rms(frame)
playbackActiveOverride?.let { reportedActive ->
// A renderer marks playback just before its first write so speaker
// output cannot enter calibration. Do not let a provider that has
// not observed the first audible frame yet undo that protection
// during the configured grace window.
val withinStartupGrace = playbackActive && playbackStartedAtMs?.let {
nowMs - it < playbackGraceMs
} == true
if (reportedActive || !withinStartupGrace) {
updatePlaybackPhase(active = reportedActive, nowMs = nowMs)
}
}
val playback = playbackActive
var justCalibrated = false
if (!playback && !calibrated) {
addAmbient(rms)
if (ambient.size >= calibrationFrames) {
freezeCalibration()
justCalibrated = true
}
}
if (!playback && (!calibrated || justCalibrated)) {
return RmsGateResult(
maybeSpeech = false,
detected = false,
rms = rms,
floor = quietFloor,
threshold = (quietFloor * thresholdMultiplier).coerceIn(
MIN_GENERATION_THRESHOLD_RMS,
MAX_THRESHOLD_RMS,
),
calibrating = !calibrated,
playbackGrace = false,
playback = false,
)
}
var threshold = if (playback) {
(quietFloor * thresholdMultiplier).coerceIn(
MIN_PLAYBACK_THRESHOLD_RMS,
MAX_THRESHOLD_RMS,
)
} else {
(quietFloor * thresholdMultiplier).coerceIn(
MIN_GENERATION_THRESHOLD_RMS,
MAX_THRESHOLD_RMS,
)
}
// Match upstream ambient drift: after initial calibration, keep the
// 90th-percentile floor current only while the room is quiet and no
// playback can contaminate it.
if (!playback && calibrated && !justCalibrated && rms < threshold) {
addAmbient(rms)
quietFloor = robustFloor(ambient)
threshold = (quietFloor * thresholdMultiplier).coerceIn(
MIN_GENERATION_THRESHOLD_RMS,
MAX_THRESHOLD_RMS,
)
}
val inPlaybackGrace = playbackStartedAtMs?.let { nowMs - it < playbackGraceMs } == true
val aboveRaw = rawSpeech && rms >= threshold && !inPlaybackGrace
val aboveConfirmed = confirmedSpeech && rms >= threshold && !inPlaybackGrace
decisions.addLast(aboveConfirmed)
while (decisions.size > decisionWindowFrames) decisions.removeAt(0)
val required = (decisionWindowFrames * requiredWindowRatio).roundToInt().coerceAtLeast(1)
val detected = aboveConfirmed && decisions.count { it } >= required
return RmsGateResult(
maybeSpeech = aboveRaw,
detected = detected,
rms = rms,
floor = quietFloor,
threshold = threshold,
calibrating = !playback && !calibrated,
playbackGrace = inPlaybackGrace,
playback = playback,
)
}
private fun freezeCalibration() {
if (!calibrated) {
quietFloor = robustFloor(ambient)
calibrated = true
}
}
private fun updatePlaybackPhase(active: Boolean, nowMs: Long) {
if (active == playbackActive) return
if (active) {
freezeCalibration()
val gapMs = playbackStoppedAtMs?.let { nowMs - it }
playbackStartedAtMs = if (gapMs == null || gapMs >= PLAYBACK_GRACE_REARM_GAP_MS) {
nowMs
} else {
null
}
playbackActive = true
decisions.clear()
} else {
playbackActive = false
playbackStartedAtMs = null
playbackStoppedAtMs = nowMs
decisions.clear()
}
}
private fun addAmbient(rms: Float) {
ambient.addLast(rms)
while (ambient.size > MAX_AMBIENT_FRAMES) ambient.removeAt(0)
}
private fun robustFloor(values: Collection<Float>): Float {
if (values.isEmpty()) return DEFAULT_QUIET_FLOOR_RMS
val sorted = values.sorted()
val percentileIndex = (ceil(sorted.size * 0.9).toInt() - 1).coerceIn(sorted.indices)
return sorted[percentileIndex].coerceAtLeast(MIN_QUIET_FLOOR_RMS)
}
private fun rms(frame: ShortArray): Float {
if (frame.isEmpty()) return 0f
var sum = 0.0
frame.forEach { sample ->
val value = sample.toDouble()
sum += value * value
}
return sqrt(sum / frame.size).toFloat()
}
companion object {
const val DEFAULT_THRESHOLD_MULTIPLIER = 3f
const val DEFAULT_PLAYBACK_GRACE_MS = 500L
internal const val DEFAULT_CALIBRATION_FRAMES = 14
internal const val DEFAULT_DECISION_WINDOW_FRAMES = 10
internal const val DEFAULT_REQUIRED_WINDOW_RATIO = 0.8f
internal const val MIN_PLAYBACK_THRESHOLD_RMS = 1_500f
internal const val MAX_THRESHOLD_RMS = 4_000f
internal const val MIN_GENERATION_THRESHOLD_RMS = 400f
internal const val DEFAULT_QUIET_FLOOR_RMS = 200f
internal const val MIN_QUIET_FLOOR_RMS = 200f
internal const val MAX_AMBIENT_FRAMES = 100
internal const val PLAYBACK_GRACE_REARM_GAP_MS = 1_000L
internal const val MIN_THRESHOLD_MULTIPLIER = 1f
internal const val MAX_THRESHOLD_MULTIPLIER = 8f
}
}
internal data class RmsGateResult(
val maybeSpeech: Boolean,
val detected: Boolean,
val rms: Float,
val floor: Float,
val threshold: Float,
val calibrating: Boolean,
val playbackGrace: Boolean,
val playback: Boolean,
)
@@ -8,6 +8,9 @@ import android.media.MediaRecorder
import android.media.audiofx.AcousticEchoCanceler
import android.media.audiofx.NoiseSuppressor
import android.util.Log
import com.hermesandroid.relay.wake.MicrophoneLease
import com.hermesandroid.relay.wake.MicrophoneOwner
import com.hermesandroid.relay.wake.MicrophoneOwnershipCoordinator
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
@@ -57,6 +60,7 @@ class VoiceRecorder(
private val bufferLock = Any()
private val stopRequested = AtomicBoolean(false)
private var audioRecord: AudioRecord? = null
private var microphoneLease: MicrophoneLease? = null
private var echoCanceler: AcousticEchoCanceler? = null
private var noiseSuppressor: NoiseSuppressor? = null
private var currentOutputFile: File? = null
@@ -79,12 +83,21 @@ class VoiceRecorder(
releaseRecorder()
}
}
val lease = MicrophoneOwnershipCoordinator.tryAcquire(MicrophoneOwner.VoiceCapture)
?: throw IllegalStateException("Microphone is in use by another voice feature")
microphoneLease = lease
val minBuffer = AudioRecord.getMinBufferSize(
val minBuffer = try {
AudioRecord.getMinBufferSize(
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
).coerceAtLeast(SAMPLE_RATE / 10 * BYTES_PER_SAMPLE)
).coerceAtLeast(SAMPLE_RATE / 10 * BYTES_PER_SAMPLE)
} catch (t: Throwable) {
MicrophoneOwnershipCoordinator.release(lease)
microphoneLease = null
throw t
}
val outFile = File(context.cacheDir, "voice_rec_${System.currentTimeMillis()}.wav")
currentOutputFile = outFile
@@ -95,21 +108,29 @@ class VoiceRecorder(
stopRequested.set(false)
_amplitude.value = 0f
val recorder = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(SAMPLE_RATE)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
.setBufferSizeInBytes(minBuffer * 2)
.build()
val recorder = try {
AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(SAMPLE_RATE)
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.build()
)
.setBufferSizeInBytes(minBuffer * 2)
.build()
} catch (t: Throwable) {
MicrophoneOwnershipCoordinator.release(lease)
microphoneLease = null
throw t
}
if (recorder.state != AudioRecord.STATE_INITIALIZED) {
recorder.release()
currentOutputFile = null
MicrophoneOwnershipCoordinator.release(lease)
microphoneLease = null
throw IllegalStateException("AudioRecord failed to initialize")
}
@@ -118,6 +139,8 @@ class VoiceRecorder(
} catch (e: Exception) {
recorder.release()
currentOutputFile = null
MicrophoneOwnershipCoordinator.release(lease)
microphoneLease = null
throw e
}
@@ -281,6 +304,8 @@ class VoiceRecorder(
try { record.release() } catch (_: Exception) { }
}
audioRecord = null
microphoneLease?.let(MicrophoneOwnershipCoordinator::release)
microphoneLease = null
readThread = null
readDone = null
}
@@ -19,6 +19,8 @@ import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
@@ -48,6 +50,7 @@ data class ConnectionAuthSecrets(
val refreshToken: String? = null,
val deviceId: String? = null,
val apiKey: String? = null,
val profileApiKeys: Map<String, String> = emptyMap(),
val pairedSessionMetaJson: String? = null,
)
@@ -114,6 +117,7 @@ class AuthManager(
private const val KEY_REFRESH_TOKEN = "refresh_token"
private const val KEY_DEVICE_ID = "device_id"
private const val KEY_API_KEY = "api_server_key"
private const val KEY_PROFILE_API_KEYS = "profile_api_server_keys"
private const val HINT_API_KEY_PRESENT = "api_key_present"
private const val KEY_PAIRED_META = "paired_session_meta_json"
// Marker (in the connection-0 token store) recording that the one-shot
@@ -132,6 +136,29 @@ class AuthManager(
*/
const val CONNECTION_ID_LEGACY: String = "legacy"
internal fun encodeProfileApiKeys(keys: Map<String, String>): String =
Json.encodeToString(
keys.mapNotNull { (profile, key) ->
val normalizedProfile = profile.trim()
val normalizedKey = key.trim()
if (normalizedProfile.isBlank() || normalizedKey.isBlank()) null
else normalizedProfile to normalizedKey
}.toMap(),
)
internal fun decodeProfileApiKeys(raw: String?): Map<String, String> {
if (raw.isNullOrBlank()) return emptyMap()
return runCatching { Json.decodeFromString<Map<String, String>>(raw) }
.getOrDefault(emptyMap())
.mapNotNull { (profile, key) ->
val normalizedProfile = profile.trim()
val normalizedKey = key.trim()
if (normalizedProfile.isBlank() || normalizedKey.isBlank()) null
else normalizedProfile to normalizedKey
}
.toMap()
}
internal fun shouldPreservePairedSessionOnAuthFail(
currentState: AuthState,
rawReason: String,
@@ -173,6 +200,7 @@ class AuthManager(
refreshToken = store.getString(KEY_REFRESH_TOKEN),
deviceId = store.getString(KEY_DEVICE_ID),
apiKey = store.getString(KEY_API_KEY),
profileApiKeys = decodeProfileApiKeys(store.getString(KEY_PROFILE_API_KEYS)),
pairedSessionMetaJson = store.getString(KEY_PAIRED_META),
)
}
@@ -188,6 +216,11 @@ class AuthManager(
writeOrRemove(store, KEY_REFRESH_TOKEN, secrets.refreshToken)
writeOrRemove(store, KEY_DEVICE_ID, secrets.deviceId)
writeOrRemove(store, KEY_API_KEY, secrets.apiKey)
writeOrRemove(
store,
KEY_PROFILE_API_KEYS,
secrets.profileApiKeys.takeIf { it.isNotEmpty() }?.let(::encodeProfileApiKeys),
)
writeOrRemove(store, KEY_PAIRED_META, secrets.pairedSessionMetaJson)
}
}
@@ -290,6 +323,7 @@ class AuthManager(
private var _store: SessionTokenStore? = null
private val storeMutex = Mutex()
private val profileApiKeysMutex = Mutex()
/**
* The encrypted-store filename for this connection — shared by [store]
@@ -416,6 +450,7 @@ class AuthManager(
KEY_REFRESH_TOKEN,
KEY_DEVICE_ID,
KEY_API_KEY,
KEY_PROFILE_API_KEYS,
KEY_PAIRED_META,
)
var migrated = false
@@ -947,6 +982,27 @@ class AuthManager(
recordApiKeyHint(false)
}
suspend fun getProfileApiKey(profileName: String): String? =
decodeProfileApiKeys(store().getString(KEY_PROFILE_API_KEYS))[profileName.trim()]
suspend fun setProfileApiKey(profileName: String, key: String) {
val normalizedProfile = profileName.trim()
require(normalizedProfile.isNotBlank()) { "Profile name must not be blank" }
profileApiKeysMutex.withLock {
val tokenStore = store()
val keys = decodeProfileApiKeys(tokenStore.getString(KEY_PROFILE_API_KEYS)).toMutableMap()
val normalizedKey = key.trim()
if (normalizedKey.isBlank()) keys.remove(normalizedProfile)
else keys[normalizedProfile] = normalizedKey
if (keys.isEmpty()) tokenStore.remove(KEY_PROFILE_API_KEYS)
else tokenStore.putString(KEY_PROFILE_API_KEYS, encodeProfileApiKeys(keys))
}
}
suspend fun clearProfileApiKey(profileName: String) {
setProfileApiKey(profileName, "")
}
val isPaired: Boolean
get() = _authState.value is AuthState.Paired
@@ -12,6 +12,7 @@ enum class AppLanguage(val languageTag: String) {
JAPANESE("ja"),
SIMPLIFIED_CHINESE("zh-Hans"),
SPANISH("es"),
RUSSIAN("ru"),
;
fun toLocaleList(): LocaleListCompat = if (languageTag.isEmpty()) {
@@ -35,6 +36,7 @@ enum class AppLanguage(val languageTag: String) {
"es" -> SPANISH
"ja" -> JAPANESE
"pt" -> BRAZILIAN_PORTUGUESE
"ru" -> RUSSIAN
"zh" -> {
val simplified = locale.script.equals("Hans", ignoreCase = true) ||
locale.script.isEmpty() ||
@@ -5,6 +5,8 @@ import androidx.datastore.core.DataStore
import androidx.datastore.preferences.core.Preferences
import androidx.datastore.preferences.core.booleanPreferencesKey
import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.floatPreferencesKey
import androidx.datastore.preferences.core.longPreferencesKey
import androidx.datastore.preferences.core.stringPreferencesKey
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.distinctUntilChanged
@@ -15,15 +17,14 @@ import kotlinx.coroutines.flow.map
*
* Phase V follow-on — owned by the voice-barge-in plan (Wave 1 / unit B1).
*
* Barge-in lets the user interrupt TTS playback by speaking. The three knobs
* Barge-in lets the user interrupt generation or TTS playback by speaking.
* here back the Voice Settings "Interruption" section added by B5 and are
* consumed by [com.hermesandroid.relay.viewmodel.VoiceViewModel] (wired in
* B4):
*
* - [enabled] — master toggle for the whole barge-in path. When false, the
* listener never starts and TTS plays uninterrupted. Default off at launch
* on both flavors so existing users aren't surprised by mic activation
* during a speaking turn.
* listener never starts and TTS plays uninterrupted. Default on matches
* upstream Hermes full-duplex voice; users can opt out here.
*
* - [sensitivity] — maps to Silero VAD threshold + hysteresis tuning inside
* [com.hermesandroid.relay.audio.VadEngine]. [BargeInSensitivity.Off] is
@@ -36,6 +37,12 @@ import kotlinx.coroutines.flow.map
* barge-in behaves like a hard cancel, which is more abrupt than most
* conversational UX expects.
*
* - [thresholdMultiplier] / [playbackGraceMs] — upstream-compatible RMS
* tuning. Defaults are 3x over the calibrated quiet floor and 500 ms.
*
* - [debugDiagnostics] — opt-in per-block VAD decision logging for logcat,
* equivalent to upstream's HERMES_VOICE_DEBUG switch.
*
* Matches the [BridgePreferences] / [VoicePreferences] / [MediaSettings] style:
* single shared DataStore (`relayDataStore`), one key per scalar field, enum
* stored as its `name` (cheap + schema-evolvable via fall-back to default on
@@ -45,6 +52,9 @@ data class BargeInPreferences(
val enabled: Boolean = DEFAULT_ENABLED,
val sensitivity: BargeInSensitivity = DEFAULT_SENSITIVITY,
val resumeAfterInterruption: Boolean = DEFAULT_RESUME_AFTER_INTERRUPTION,
val thresholdMultiplier: Float = DEFAULT_THRESHOLD_MULTIPLIER,
val playbackGraceMs: Long = DEFAULT_PLAYBACK_GRACE_MS,
val debugDiagnostics: Boolean = DEFAULT_DEBUG_DIAGNOSTICS,
)
/**
@@ -62,9 +72,12 @@ enum class BargeInSensitivity {
High,
}
const val DEFAULT_ENABLED: Boolean = false
const val DEFAULT_ENABLED: Boolean = true
val DEFAULT_SENSITIVITY: BargeInSensitivity = BargeInSensitivity.Default
const val DEFAULT_RESUME_AFTER_INTERRUPTION: Boolean = true
const val DEFAULT_THRESHOLD_MULTIPLIER: Float = 3f
const val DEFAULT_PLAYBACK_GRACE_MS: Long = 500L
const val DEFAULT_DEBUG_DIAGNOSTICS: Boolean = false
/**
* DataStore-backed repository for [BargeInPreferences].
@@ -86,6 +99,10 @@ class BargeInPreferencesRepository(
internal val KEY_SENSITIVITY = stringPreferencesKey("barge_in_sensitivity")
internal val KEY_RESUME_AFTER_INTERRUPTION =
booleanPreferencesKey("barge_in_resume_after_interruption")
internal val KEY_THRESHOLD_MULTIPLIER =
floatPreferencesKey("barge_in_threshold_multiplier")
internal val KEY_PLAYBACK_GRACE_MS = longPreferencesKey("barge_in_playback_grace_ms")
internal val KEY_DEBUG_DIAGNOSTICS = booleanPreferencesKey("barge_in_debug_diagnostics")
}
val flow: Flow<BargeInPreferences> = dataStore.data
@@ -96,6 +113,14 @@ class BargeInPreferencesRepository(
?: DEFAULT_SENSITIVITY,
resumeAfterInterruption = prefs[KEY_RESUME_AFTER_INTERRUPTION]
?: DEFAULT_RESUME_AFTER_INTERRUPTION,
thresholdMultiplier = prefs[KEY_THRESHOLD_MULTIPLIER]
?.coerceIn(MIN_THRESHOLD_MULTIPLIER, MAX_THRESHOLD_MULTIPLIER)
?: DEFAULT_THRESHOLD_MULTIPLIER,
playbackGraceMs = prefs[KEY_PLAYBACK_GRACE_MS]
?.coerceIn(MIN_PLAYBACK_GRACE_MS, MAX_PLAYBACK_GRACE_MS)
?: DEFAULT_PLAYBACK_GRACE_MS,
debugDiagnostics = prefs[KEY_DEBUG_DIAGNOSTICS]
?: DEFAULT_DEBUG_DIAGNOSTICS,
)
}
.distinctUntilChanged()
@@ -112,6 +137,33 @@ class BargeInPreferencesRepository(
dataStore.edit { it[KEY_RESUME_AFTER_INTERRUPTION] = value }
}
suspend fun setThresholdMultiplier(value: Float) {
dataStore.edit {
it[KEY_THRESHOLD_MULTIPLIER] = value.coerceIn(
MIN_THRESHOLD_MULTIPLIER,
MAX_THRESHOLD_MULTIPLIER,
)
}
}
suspend fun setPlaybackGraceMs(value: Long) {
dataStore.edit {
it[KEY_PLAYBACK_GRACE_MS] = value.coerceIn(
MIN_PLAYBACK_GRACE_MS,
MAX_PLAYBACK_GRACE_MS,
)
}
}
suspend fun setDebugDiagnostics(value: Boolean) {
dataStore.edit { it[KEY_DEBUG_DIAGNOSTICS] = value }
}
private fun decodeSensitivity(raw: String): BargeInSensitivity =
runCatching { BargeInSensitivity.valueOf(raw) }.getOrDefault(DEFAULT_SENSITIVITY)
}
private const val MIN_THRESHOLD_MULTIPLIER = 1f
private const val MAX_THRESHOLD_MULTIPLIER = 8f
private const val MIN_PLAYBACK_GRACE_MS = 0L
private const val MAX_PLAYBACK_GRACE_MS = 3_000L
@@ -247,6 +247,7 @@ data class Connection(
apiServerUrl: String,
relayUrl: String,
extraApiUrls: List<Pair<String, String>> = emptyList(),
dashboardUrl: String? = null,
): List<EndpointCandidate> {
val routes = buildList {
endpointCandidateFromApiUrl(
@@ -255,6 +256,7 @@ data class Connection(
apiServerUrl = apiServerUrl,
relayUrl = relayUrl.takeIf { it.isNotBlank() }
?: deriveDefaultRelayUrl(apiServerUrl).orEmpty(),
dashboardUrl = dashboardUrl,
)?.let(::add)
extraApiUrls
@@ -266,6 +268,7 @@ data class Connection(
priority = index + 1,
apiServerUrl = url,
relayUrl = deriveDefaultRelayUrl(url).orEmpty(),
dashboardUrl = dashboardUrl,
)?.let(::add)
}
}
@@ -340,6 +343,7 @@ data class Connection(
priority: Int,
apiServerUrl: String,
relayUrl: String,
dashboardUrl: String? = null,
): EndpointCandidate? {
val uri = runCatching { URI(apiServerUrl.trim().trimEnd('/')) }.getOrNull()
?: return null
@@ -363,12 +367,62 @@ data class Connection(
role = role.ifBlank { inferRouteRole(apiServerUrl) },
priority = priority,
api = ApiEndpoint(host = host, port = port, tls = tls),
dashboard = deriveDefaultDashboardUrl(apiServerUrl)
dashboard = dashboardUrl
?.trim()
?.trimEnd('/')
?.takeIf { it.isNotBlank() && urlsShareHost(it, apiServerUrl) }
?.let { DashboardEndpoint(url = it) }
?: deriveDefaultDashboardUrl(apiServerUrl)
?.let { DashboardEndpoint(url = it) },
relay = RelayEndpoint(url = resolvedRelayUrl, transportHint = transportHint),
)
}
/**
* Reconcile stored API-derived routes with the Dashboard origin that
* was actually verified during setup. Older app versions synthesized
* `:9119` for every API route, even when the same host was reached
* through an HTTPS reverse proxy on 443. Replace only that conventional
* synthesized value (or a missing value); preserve explicit and
* different-host LAN/Tailscale routes.
*/
fun reconcileDashboardRoutes(
dashboardUrl: String?,
candidates: List<EndpointCandidate>,
): List<EndpointCandidate> {
val explicitDashboard = dashboardUrl
?.trim()
?.trimEnd('/')
?.takeIf { it.isNotBlank() }
?: return candidates
return candidates.map { candidate ->
val apiUrl = candidate.api?.url ?: return@map candidate
if (!urlsShareHost(explicitDashboard, apiUrl)) return@map candidate
val currentDashboard = candidate.dashboard?.url
val derivedDashboard = deriveDefaultDashboardUrl(apiUrl)
val canReplace = currentDashboard.isNullOrBlank() ||
(
derivedDashboard != null &&
currentDashboard.trim().trimEnd('/')
.equals(derivedDashboard, ignoreCase = true)
)
if (canReplace) {
candidate.copy(dashboard = DashboardEndpoint(url = explicitDashboard))
} else {
candidate
}
}
}
fun urlsShareHost(leftUrl: String, rightUrl: String): Boolean {
val leftHost = runCatching { URI(leftUrl.trim()) }.getOrNull()?.host
val rightHost = runCatching { URI(rightUrl.trim()) }.getOrNull()?.host
return !leftHost.isNullOrBlank() &&
!rightHost.isNullOrBlank() &&
leftHost.equals(rightHost, ignoreCase = true)
}
/**
* De-duplication identity for rebuilding stored routes. Prefer the
* legacy API authority when present so an older API-only candidate and
@@ -543,29 +543,6 @@ class ConnectionStore private constructor(
}
}
private fun Connection.withDashboardDefaults(): Connection {
val derivedDashboardUrl = Connection.deriveDefaultDashboardUrl(apiServerUrl)
val normalizedRoutes = routeCandidates.ifEmpty {
Connection.buildRouteCandidates(apiServerUrl, relayUrl)
}
val normalizedPreferredRouteRole = preferredRouteRole?.takeIf { preferred ->
normalizedRoutes.any { it.role.equals(preferred, ignoreCase = true) }
}
return if (
(dashboardUrl.isNullOrBlank() && derivedDashboardUrl != null) ||
normalizedRoutes != routeCandidates ||
normalizedPreferredRouteRole != preferredRouteRole
) {
copy(
dashboardUrl = dashboardUrl?.takeIf { it.isNotBlank() } ?: derivedDashboardUrl,
routeCandidates = normalizedRoutes,
preferredRouteRole = normalizedPreferredRouteRole,
)
} else {
this
}
}
companion object {
private const val TAG = "ConnectionStore"
@@ -585,3 +562,40 @@ class ConnectionStore private constructor(
private const val DEFAULT_RELAY_URL = "ws://localhost:8767"
}
}
/**
* Restore route defaults after loading a serialized connection. This remains
* internal so focused persistence tests can exercise the same normalization
* path used by [ConnectionStore].
*/
internal fun Connection.withDashboardDefaults(): Connection {
val derivedDashboardUrl = Connection.deriveDefaultDashboardUrl(apiServerUrl)
val effectiveDashboardUrl = dashboardUrl?.takeIf { it.isNotBlank() } ?: derivedDashboardUrl
val storedOrDefaultRoutes = routeCandidates.ifEmpty {
Connection.buildRouteCandidates(
apiServerUrl = apiServerUrl,
relayUrl = relayUrl,
dashboardUrl = effectiveDashboardUrl,
)
}
val normalizedRoutes = Connection.reconcileDashboardRoutes(
dashboardUrl = effectiveDashboardUrl,
candidates = storedOrDefaultRoutes,
)
val normalizedPreferredRouteRole = preferredRouteRole?.takeIf { preferred ->
normalizedRoutes.any { it.role.equals(preferred, ignoreCase = true) }
}
return if (
dashboardUrl != effectiveDashboardUrl ||
normalizedRoutes != routeCandidates ||
normalizedPreferredRouteRole != preferredRouteRole
) {
copy(
dashboardUrl = effectiveDashboardUrl,
routeCandidates = normalizedRoutes,
preferredRouteRole = normalizedPreferredRouteRole,
)
} else {
this
}
}
@@ -22,6 +22,7 @@ object FeatureFlags {
// DataStore keys
private val KEY_DEV_OPTIONS_UNLOCKED = booleanPreferencesKey("dev_options_unlocked")
private val KEY_PET_TERRAIN_OVERLAY = booleanPreferencesKey("pet_terrain_overlay")
/** Whether the app is running a debug build. */
val isDevBuild: Boolean get() = BuildConfig.DEV_MODE
@@ -32,6 +33,35 @@ object FeatureFlags {
prefs[KEY_DEV_OPTIONS_UNLOCKED] ?: isDevBuild
}
/**
* Developer-only visualization of the floating pet's measured terrain.
*
* The persisted request is intentionally weaker than both gates: release
* builds can never enable it, and explicitly locking Developer Options
* suppresses it immediately.
*/
fun petTerrainOverlayEnabled(context: Context): Flow<Boolean> =
context.relayDataStore.data.map { prefs ->
petTerrainOverlayEffective(
isDevBuild = isDevBuild,
devOptionsUnlocked = prefs[KEY_DEV_OPTIONS_UNLOCKED] ?: isDevBuild,
requested = prefs[KEY_PET_TERRAIN_OVERLAY] ?: false,
)
}
internal fun petTerrainOverlayEffective(
isDevBuild: Boolean,
devOptionsUnlocked: Boolean,
requested: Boolean,
): Boolean = isDevBuild && devOptionsUnlocked && requested
/** Persist the developer's overlay request. Runtime gates remain authoritative. */
suspend fun setPetTerrainOverlayEnabled(context: Context, enabled: Boolean) {
context.relayDataStore.edit { prefs ->
prefs[KEY_PET_TERRAIN_OVERLAY] = enabled
}
}
/** Unlock Developer Options. */
suspend fun unlockDevOptions(context: Context) {
context.relayDataStore.edit { prefs ->
@@ -43,6 +73,7 @@ object FeatureFlags {
suspend fun lockDevOptions(context: Context) {
context.relayDataStore.edit { prefs ->
prefs[KEY_DEV_OPTIONS_UNLOCKED] = false
prefs[KEY_PET_TERRAIN_OVERLAY] = false
}
}
@@ -0,0 +1,135 @@
package com.hermesandroid.relay.data
import android.content.Context
import androidx.datastore.core.DataStore
import androidx.datastore.preferences.core.Preferences
import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.floatPreferencesKey
import androidx.datastore.preferences.core.intPreferencesKey
import androidx.datastore.preferences.core.stringPreferencesKey
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.flow.map
/** Exact cadence values consumed by the floating-pet behavior director. */
data class PetBehaviorPacing(
val responseVisitDelayMs: Long,
val roamIntervalMs: Long,
val idleReactionCadenceMs: Long,
) {
init {
require(responseVisitDelayMs > 0L)
require(roamIntervalMs > responseVisitDelayMs)
require(idleReactionCadenceMs > roamIntervalMs)
}
}
/**
* User-facing pet activity presets.
*
* These values control how often an otherwise-idle pet may act. Existing
* animation, reduced-motion, touch-exploration, scrolling, and agent-activity
* gates remain authoritative and may always defer an action.
*/
enum class PetTemperament(val pacing: PetBehaviorPacing) {
Calm(
PetBehaviorPacing(
responseVisitDelayMs = 2_500L,
roamIntervalMs = 12_000L,
idleReactionCadenceMs = 28_000L,
),
),
Balanced(
PetBehaviorPacing(
responseVisitDelayMs = 1_500L,
roamIntervalMs = 8_000L,
idleReactionCadenceMs = 18_000L,
),
),
Playful(
PetBehaviorPacing(
responseVisitDelayMs = 750L,
roamIntervalMs = 5_000L,
idleReactionCadenceMs = 10_000L,
),
),
}
val DEFAULT_PET_TEMPERAMENT: PetTemperament = PetTemperament.Balanced
const val DEFAULT_PET_SIZE_SCALE: Float = 1f
const val MIN_PET_SIZE_SCALE: Float = 0.6f
const val MAX_PET_SIZE_SCALE: Float = 1.2f
private const val LEGACY_PET_SIZE_BASE_SCALE: Float = 1.25f
private const val CURRENT_PET_SIZE_SCALE_VERSION: Int = 2
internal fun sanitizedPetSizeScale(value: Float?): Float =
value?.takeIf(Float::isFinite)?.coerceIn(MIN_PET_SIZE_SCALE, MAX_PET_SIZE_SCALE)
?: DEFAULT_PET_SIZE_SCALE
internal fun decodeStoredPetSizeScale(value: Float?, version: Int?): Float {
if (value == null) return DEFAULT_PET_SIZE_SCALE
val rebased = if (version == null) value / LEGACY_PET_SIZE_BASE_SCALE else value
return sanitizedPetSizeScale(rebased)
}
data class PetBehaviorPreferences(
val temperament: PetTemperament = DEFAULT_PET_TEMPERAMENT,
val sizeScale: Float = DEFAULT_PET_SIZE_SCALE,
) {
/**
* Runtime seam for the behavior director. A disabled motion gate returns
* no pacing rather than weakening the app's accessibility policy.
*/
fun pacingWhenMotionAllowed(motionAllowed: Boolean): PetBehaviorPacing? =
temperament.pacing.takeIf { motionAllowed }
}
/** Additive, phone-local DataStore persistence for pet behavior preferences. */
class PetBehaviorPreferencesRepository(
private val dataStore: DataStore<Preferences>,
) {
constructor(context: Context) : this(context.relayDataStore)
companion object {
internal val KEY_TEMPERAMENT = stringPreferencesKey("pet_temperament")
internal val KEY_SIZE_SCALE = floatPreferencesKey("pet_size_scale")
internal val KEY_SIZE_SCALE_VERSION = intPreferencesKey("pet_size_scale_version")
}
val flow: Flow<PetBehaviorPreferences> = dataStore.data
.map { preferences ->
PetBehaviorPreferences(
temperament = decodeTemperament(preferences[KEY_TEMPERAMENT]),
sizeScale = decodeStoredPetSizeScale(
value = preferences[KEY_SIZE_SCALE],
version = preferences[KEY_SIZE_SCALE_VERSION],
),
)
}
.distinctUntilChanged()
val temperament: Flow<PetTemperament> = flow
.map { preferences -> preferences.temperament }
.distinctUntilChanged()
val sizeScale: Flow<Float> = flow
.map { preferences -> preferences.sizeScale }
.distinctUntilChanged()
suspend fun setTemperament(temperament: PetTemperament) {
dataStore.edit { preferences ->
preferences[KEY_TEMPERAMENT] = temperament.name
}
}
suspend fun setSizeScale(sizeScale: Float) {
dataStore.edit { preferences ->
preferences[KEY_SIZE_SCALE] = sanitizedPetSizeScale(sizeScale)
preferences[KEY_SIZE_SCALE_VERSION] = CURRENT_PET_SIZE_SCALE_VERSION
}
}
private fun decodeTemperament(raw: String?): PetTemperament =
raw?.let { stored -> PetTemperament.entries.firstOrNull { it.name == stored } }
?: DEFAULT_PET_TEMPERAMENT
}
@@ -47,9 +47,10 @@ import kotlinx.serialization.Serializable
*
* **Hermes profile API metadata.** A relay can advertise an isolated
* profile API server without exposing its secret. When [apiServerUrl] is
* present, Android routes chat/session traffic to that URL and reuses the
* active connection's stored API key. Operators that use distinct API keys
* per profile should pair those profile API servers as separate connections.
* present, Android routes chat/session traffic to that URL using the active
* connection credential. A positively identified shared multiplex
* `/p/<profile>` route instead uses a separately encrypted profile credential;
* the root connection key is never reused for that route.
*/
@Serializable
data class Profile(
@@ -13,6 +13,11 @@ import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
val DEFAULT_VOICE_STOP_PHRASES: List<String> = listOf("stop")
/**
* User-tunable voice mode preferences.
@@ -36,6 +41,16 @@ data class VoiceSettings(
val audioRoute: String = VoiceAudioRoute.Auto.storageValue,
val interactionMode: String = "tap",
val silenceThresholdMs: Long = 1250L,
/** Exact phrases that end an active voice chat; empty disables the command. */
val stopPhrases: List<String> = DEFAULT_VOICE_STOP_PHRASES,
/**
* When true, voice keeps progress visual and waits for the settled Hermes
* answer before speaking. Tool status, service updates, and intermediate
* assistant commentary are not narrated.
*/
val finalAnswerOnly: Boolean = false,
/** Presentation only; changing this never restarts or interrupts voice. */
val presentationMode: String = VoicePresentationMode.Focus.storageValue,
val realtimeTraceDetails: Boolean = false,
/**
* When true (default), Realtime Agent keeps one provider session/socket open
@@ -122,6 +137,16 @@ enum class VoiceAudioRoute(val storageValue: String) {
}
}
enum class VoicePresentationMode(val storageValue: String) {
Focus("focus"),
Conversation("conversation");
companion object {
fun fromStorage(value: String?): VoicePresentationMode =
values().firstOrNull { it.storageValue == value } ?: Focus
}
}
/**
* Active scope for per-profile voice prefs.
*
@@ -182,6 +207,9 @@ class VoicePreferencesRepository(private val dataStore: DataStore<Preferences>)
// un-namespaced means switching profiles never churns these.
private val KEY_INTERACTION_MODE = stringPreferencesKey("voice_interaction_mode")
private val KEY_SILENCE_THRESHOLD_MS = longPreferencesKey("voice_silence_threshold_ms")
private val KEY_STOP_PHRASES = stringPreferencesKey("voice_stop_phrases")
private val KEY_FINAL_ANSWER_ONLY = booleanPreferencesKey("voice_final_answer_only")
private val KEY_PRESENTATION_MODE = stringPreferencesKey("voice_presentation_mode")
private val KEY_REALTIME_TRACE_DETAILS = booleanPreferencesKey("voice_realtime_trace_details")
private val KEY_REALTIME_PERSISTENT_SESSION =
booleanPreferencesKey("voice_realtime_persistent_session")
@@ -191,8 +219,14 @@ class VoicePreferencesRepository(private val dataStore: DataStore<Preferences>)
const val DEFAULT_INTERACTION_MODE = "tap"
// 1250 ms matches hermes-desktop voice_mode `silenceMs` end-of-speech.
const val DEFAULT_SILENCE_THRESHOLD_MS = 1250L
const val DEFAULT_FINAL_ANSWER_ONLY = false
const val DEFAULT_PRESENTATION_MODE = "focus"
const val DEFAULT_REALTIME_TRACE_DETAILS = false
const val DEFAULT_REALTIME_PERSISTENT_SESSION = true
private val stopPhrasesJson = Json {
ignoreUnknownKeys = true
isLenient = true
}
/**
* Build the storage name for a per-profile [base] key under [scope].
@@ -258,6 +292,11 @@ class VoicePreferencesRepository(private val dataStore: DataStore<Preferences>)
// --- global (shared across profiles) ---
interactionMode = prefs[KEY_INTERACTION_MODE] ?: DEFAULT_INTERACTION_MODE,
silenceThresholdMs = prefs[KEY_SILENCE_THRESHOLD_MS] ?: DEFAULT_SILENCE_THRESHOLD_MS,
stopPhrases = decodeStopPhrases(prefs[KEY_STOP_PHRASES]),
finalAnswerOnly = prefs[KEY_FINAL_ANSWER_ONLY] ?: DEFAULT_FINAL_ANSWER_ONLY,
presentationMode = VoicePresentationMode.fromStorage(
prefs[KEY_PRESENTATION_MODE] ?: DEFAULT_PRESENTATION_MODE,
).storageValue,
realtimeTraceDetails = prefs[KEY_REALTIME_TRACE_DETAILS]
?: DEFAULT_REALTIME_TRACE_DETAILS,
realtimePersistentSession = prefs[KEY_REALTIME_PERSISTENT_SESSION]
@@ -367,6 +406,23 @@ class VoicePreferencesRepository(private val dataStore: DataStore<Preferences>)
dataStore.edit { it[KEY_SILENCE_THRESHOLD_MS] = ms.coerceAtLeast(500L) }
}
suspend fun setStopPhrases(phrases: List<String>) {
val normalized = phrases.asSequence()
.map(String::trim)
.filter(String::isNotEmpty)
.distinct()
.toList()
dataStore.edit { it[KEY_STOP_PHRASES] = stopPhrasesJson.encodeToString(normalized) }
}
suspend fun setFinalAnswerOnly(enabled: Boolean) {
dataStore.edit { it[KEY_FINAL_ANSWER_ONLY] = enabled }
}
suspend fun setPresentationMode(mode: VoicePresentationMode) {
dataStore.edit { it[KEY_PRESENTATION_MODE] = mode.storageValue }
}
suspend fun setRealtimeTraceDetails(enabled: Boolean) {
dataStore.edit { it[KEY_REALTIME_TRACE_DETAILS] = enabled }
}
@@ -375,6 +431,17 @@ class VoicePreferencesRepository(private val dataStore: DataStore<Preferences>)
dataStore.edit { it[KEY_REALTIME_PERSISTENT_SESSION] = enabled }
}
private fun decodeStopPhrases(raw: String?): List<String> {
if (raw == null) return DEFAULT_VOICE_STOP_PHRASES
return runCatching { stopPhrasesJson.decodeFromString<List<String>>(raw) }
.getOrDefault(DEFAULT_VOICE_STOP_PHRASES)
.asSequence()
.map(String::trim)
.filter(String::isNotEmpty)
.distinct()
.toList()
}
/**
* Atomically apply the phone-side portion of [preset]. Only fields owned by
* the preset are written, so route/provider/model/voice overrides and other
@@ -16,6 +16,7 @@ import com.hermesandroid.relay.diagnostics.DiagnosticSeverity
import com.hermesandroid.relay.diagnostics.DiagnosticsLog
import com.hermesandroid.relay.network.relay.models.Envelope
import com.hermesandroid.relay.network.shared.EndpointResolver
import com.hermesandroid.relay.network.shared.EndpointSurface
import com.hermesandroid.relay.network.shutdownOffMainThread
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -161,7 +162,7 @@ class ConnectionManager(
@Volatile
private var serverUrl: String? = null
private var reconnectAttempt = 0
private val reconnectState = RelayReconnectState()
private var shouldReconnect = true
// Last HTTP status seen during WSS upgrade, captured in onFailure.
// Used by scheduleReconnect() to pick an appropriate backoff — notably
@@ -170,14 +171,6 @@ class ConnectionManager(
@Volatile
private var lastUpgradeResponseCode: Int? = null
// Consecutive relay socket failures (response == null) since the last
// successful onOpen. One slow Tailscale/DERP cold-start handshake must not
// immediately evict the active route from the SHARED resolver cache (chat +
// dashboard ride the same resolver), so we only poison the route after a
// couple of consecutive transport-level failures.
@Volatile
private var consecutiveSocketFailures = 0
// The relay requires the FIRST frame on a socket to be `system/auth` and
// rejects the whole connection otherwise ("expected system/auth, got
// <channel>/<type>"). `authenticated` gates [send] so nothing (notably the
@@ -206,6 +199,10 @@ class ConnectionManager(
private val _activeEndpoint = MutableStateFlow<EndpointCandidate?>(null)
val activeEndpoint: StateFlow<EndpointCandidate?> = _activeEndpoint.asStateFlow()
/** Relay-only winner, deliberately separate from the standard route. */
@Volatile
private var activeRelayEndpoint: EndpointCandidate? = null
/**
* Manual role override. When non-null, the resolver's output is replaced
* with whichever candidate in the stored list matches this role (case-
@@ -262,9 +259,9 @@ class ConnectionManager(
private const val TAG = "ConnectionManager"
private const val MAX_BACKOFF_MS = 30_000L
private const val BASE_BACKOFF_MS = 1_000L
// How many consecutive relay socket failures before we mark the active
// endpoint unreachable in the shared resolver cache. Tolerates a single
// cold-start blip on a slow remote (Tailscale DERP) link.
// How many consecutive failures on one relay socket URL before we mark
// that candidate's Relay surface unreachable. Standard Dashboard/API
// reachability is cached independently and remains healthy.
private const val MARK_UNREACHABLE_AFTER_FAILURES = 2
// Settle window before re-resolving after a network event. Long
// enough to coalesce the onAvailable burst of a handoff, short
@@ -325,17 +322,19 @@ class ConnectionManager(
// behavior for freshly-upgraded installs and for v1/v2 QRs where
// the synthesized list just collapses to the same URL anyway.
scope.launch {
val resolved = resolveBestEndpointSafe()
val resolvedRelayUrl = resolved?.relay?.url?.takeIf { it.isNotBlank() }
val resolved = resolveBestEndpointSafe(EndpointSurface.Standard)
val relayResolved = resolveBestEndpointSafe(EndpointSurface.Relay)
val resolvedRelayUrl = relayResolved?.relay?.url?.takeIf { it.isNotBlank() }
val targetUrl = resolvedRelayUrl ?: url.takeIf { it.isNotBlank() }
activeRelayEndpoint = relayResolved
if (resolved != null) {
_activeEndpoint.value = resolved
Log.i(TAG, "connect: resolver picked role=${resolved.role} " +
"route=${resolved.primaryRouteUrl()} (relay fallback would have been $url)")
Log.i(TAG, "connect: standard resolver picked role=${resolved.role} " +
"route=${resolved.primaryRouteUrl()}")
DiagnosticsLog.record(
category = DiagnosticCategory.Relay,
category = DiagnosticCategory.Endpoint,
severity = DiagnosticSeverity.Info,
title = context?.getString(R.string.conn_diag_route_selected) ?: "Relay route selected",
title = context?.getString(R.string.conn_diag_route_selected) ?: "Route selected",
endpointRole = resolved.role,
url = resolved.primaryRouteUrl(),
)
@@ -350,6 +349,13 @@ class ConnectionManager(
url = url,
)
}
relayResolved?.let { relayRoute ->
Log.i(
TAG,
"connect: relay resolver picked role=${relayRoute.role} " +
"url=${relayRoute.relay?.url}",
)
}
if (targetUrl != null) {
connectToUrlOnMainPath(targetUrl)
} else {
@@ -368,6 +374,7 @@ class ConnectionManager(
private fun connectToUrlOnMainPath(
url: String,
replaceReason: String = "Relay socket replaced",
preserveReconnectBackoff: Boolean = false,
) {
val isInsecure = url.startsWith("ws://") && !url.startsWith("wss://")
if (isInsecure && !_insecureMode.value) {
@@ -429,7 +436,11 @@ class ConnectionManager(
serverUrl = normalized
shouldReconnect = true
reconnectAttempt = 0
if (preserveReconnectBackoff) {
reconnectState.beginAutomaticRouteSwap(normalized)
} else {
reconnectState.beginExplicitConnect(normalized)
}
doConnect(normalized, previousSocket, replaceReason)
}
@@ -445,9 +456,12 @@ class ConnectionManager(
* Wraps the DataStore read in a 1-second timeout; if DataStore stalls
* for any reason we don't block the connect loop forever.
*/
suspend fun resolveBestEndpoint(): EndpointCandidate? = resolveBestEndpointSafe()
suspend fun resolveBestEndpoint(): EndpointCandidate? =
resolveBestEndpointSafe(EndpointSurface.Standard)
private suspend fun resolveBestEndpointSafe(): EndpointCandidate? {
private suspend fun resolveBestEndpointSafe(
surface: EndpointSurface,
): EndpointCandidate? {
val resolver = endpointResolver ?: return null
val ctx = context ?: return null
@@ -486,14 +500,14 @@ class ConnectionManager(
}
if (preferred != null) {
// Single-element list still respects the 2s probe gate.
val winner = resolver.resolve(listOf(preferred))
val winner = resolver.resolve(listOf(preferred), surface)
if (winner != null) return winner
Log.i(TAG, "manualRoleOverride=$preferredRole not reachable — " +
"falling through to strict-priority resolve")
}
}
return resolver.resolve(endpoints)
return resolver.resolve(endpoints, surface)
}
/**
@@ -521,7 +535,8 @@ class ConnectionManager(
suspend fun probeAndReconnectNow(): EndpointCandidate? {
endpointResolver?.clearCache()
val current = serverUrl
val resolved = resolveBestEndpointSafe()
val resolved = resolveBestEndpointSafe(EndpointSurface.Standard)
val relayResolved = resolveBestEndpointSafe(EndpointSurface.Relay)
if (resolved == null && _connectionState.value == ConnectionState.Connected) {
// Transient probe miss while the relay socket is demonstrably up
// — keep the live route published rather than downgrading every
@@ -529,7 +544,8 @@ class ConnectionManager(
return _activeEndpoint.value
}
_activeEndpoint.value = resolved
val targetUrl = resolved?.relay?.url ?: current ?: return resolved
if (relayResolved != null) activeRelayEndpoint = relayResolved
val targetUrl = relayResolved?.relay?.url ?: current ?: return resolved
val normalizedTarget = normalizeRelayUrl(targetUrl)
// Reconnect when the winner changed, and also when the socket is
// stale/disconnected on the same winner. The latter makes the
@@ -566,7 +582,7 @@ class ConnectionManager(
*/
suspend fun refreshActiveEndpoint(clearProbeCache: Boolean = false): EndpointCandidate? {
if (clearProbeCache) endpointResolver?.clearCache()
val resolved = resolveBestEndpointSafe()
val resolved = resolveBestEndpointSafe(EndpointSurface.Standard)
if (resolved == null && _connectionState.value == ConnectionState.Connected) {
// Transient probe miss while the relay socket is demonstrably up
// (slow resume, mid-handoff blip) — keep publishing the live
@@ -590,9 +606,9 @@ class ConnectionManager(
fun getManualRoleOverride(): String? = _manualRoleOverride.value
private fun markActiveEndpointUnreachable(reason: String) {
val active = _activeEndpoint.value ?: return
endpointResolver?.markUnreachable(active)
private fun markActiveRelayEndpointUnreachable(reason: String) {
val active = activeRelayEndpoint ?: return
endpointResolver?.markUnreachable(active, EndpointSurface.Relay)
Log.i(TAG, "marked endpoint role=${active.role} unreachable ($reason)")
}
@@ -615,9 +631,9 @@ class ConnectionManager(
// Tailscale's tun churns onAvailable repeatedly — coalesces into a
// single cache wipe + re-probe instead of one per event. onLost
// manages its own cache (clear + markUnreachable) and passes false.
if (wipeCache) endpointResolver?.clearCache()
if (wipeCache) endpointResolver.clearCache()
val current = serverUrl
val resolved = resolveBestEndpointSafe()
val resolved = resolveBestEndpointSafe(EndpointSurface.Standard)
if (resolved == null) {
// Hysteresis for the AUTOMATIC (network-callback) path. A
// transient cold-route probe miss must NOT null the published
@@ -662,7 +678,10 @@ class ConnectionManager(
// (connectToUrlOnMainPath force-sets shouldReconnect = true, so
// the swap path never re-checked it.)
if (!shouldReconnect) return@launch
val relayUrl = resolved.relay?.url?.takeIf { it.isNotBlank() } ?: return@launch
val relayResolved = resolveBestEndpointSafe(EndpointSurface.Relay)
if (relayResolved != null) activeRelayEndpoint = relayResolved
val relayUrl = relayResolved?.relay?.url?.takeIf { it.isNotBlank() }
?: return@launch
val normalizedNew = normalizeRelayUrl(relayUrl)
if (normalizedNew != current) {
Log.i(TAG, "network change: swapping $current → $normalizedNew")
@@ -708,7 +727,9 @@ class ConnectionManager(
Log.i(TAG, "network loss sustained past grace — marking active endpoint unreachable and resolving fallback")
sustainedLossDeclared = true
endpointResolver?.clearCache()
markActiveEndpointUnreachable("network lost (sustained)")
_activeEndpoint.value?.let { active ->
endpointResolver?.markUnreachable(active, EndpointSurface.Standard)
}
// wipeCache=false: we just cleared + poisoned the dead route
// above; re-wiping inside the job would drop that negative
// entry and let the dead route win the resolve again.
@@ -779,6 +800,8 @@ class ConnectionManager(
// the ViewModel on the next connection load.
_manualRoleOverride.value = null
_activeEndpoint.value = null
activeRelayEndpoint = null
reconnectState.reset()
}
fun shutdown() {
@@ -827,7 +850,7 @@ class ConnectionManager(
return
}
_connectionState.value = if (reconnectAttempt > 0) {
_connectionState.value = if (reconnectState.reconnectAttempt > 0) {
ConnectionState.Reconnecting
} else {
ConnectionState.Connecting
@@ -884,9 +907,8 @@ class ConnectionManager(
webSocket.cancel()
return
}
reconnectAttempt = 0
reconnectState.connected(url)
lastUpgradeResponseCode = null
consecutiveSocketFailures = 0
_connectionState.value = ConnectionState.Connected
Log.i(TAG, "onOpen: WSS handshake complete ($url)")
DiagnosticsLog.record(
@@ -983,14 +1005,14 @@ class ConnectionManager(
if (response == null) {
// Transport-level failure (no HTTP upgrade response): on a
// remote (Tailscale) link the first handshake can fail cold.
// Don't evict the only working route from the shared resolver
// on a single blip — wait for it to repeat. A genuinely
// sustained network loss is handled separately by onLost.
consecutiveSocketFailures++
if (consecutiveSocketFailures >= MARK_UNREACHABLE_AFTER_FAILURES) {
markActiveEndpointUnreachable("socket failure x$consecutiveSocketFailures")
// Don't evict this Relay surface on a single blip — wait
// for the same socket URL to fail again. Standard route
// health is separate and is never poisoned here.
val failureCount = reconnectState.recordSocketFailure(url)
if (failureCount >= MARK_UNREACHABLE_AFTER_FAILURES) {
markActiveRelayEndpointUnreachable("socket failure x$failureCount")
} else {
Log.i(TAG, "relay socket failure $consecutiveSocketFailures/$MARK_UNREACHABLE_AFTER_FAILURES — not yet poisoning route")
Log.i(TAG, "relay socket failure $failureCount/$MARK_UNREACHABLE_AFTER_FAILURES — not yet poisoning route")
}
}
authenticated = false
@@ -1029,7 +1051,12 @@ class ConnectionManager(
}
val url = serverUrl ?: return
reconnectAttempt++
val reconnectAttempt = reconnectState.nextReconnectAttempt()
// Keep the socket lifecycle visibly in-flight for the whole backoff
// window. Callers such as reconnectIfStale() treat Disconnected as an
// invitation to call connect() again; leaving this state Disconnected
// let screen entry restart both the route resolve and the retry counter.
_connectionState.value = ConnectionState.Reconnecting
// Server-issued 429 means we're IP-banned — keep retrying at our
// normal exponential cadence and we'll re-fill the ban bucket on
@@ -1083,7 +1110,7 @@ class ConnectionManager(
// expires, auth state may have changed (e.g., user hit Revoke
// during the retry window).
if (shouldReconnect && reconnectGate()) {
val resolved = resolveBestEndpointSafe()
val resolved = resolveBestEndpointSafe(EndpointSurface.Relay)
val targetUrl = resolved?.relay?.url
if (resolved != null) {
// Mirror scheduleNetworkReResolve: clear the sustained-loss
@@ -1092,17 +1119,14 @@ class ConnectionManager(
// in onLost's grace job but can be cleared on EITHER success
// edge — network-callback or relay-timer.)
sustainedLossDeclared = false
_activeEndpoint.value = resolved
} else if (sustainedLossDeclared || _activeEndpoint.value == null) {
// Same hysteresis as scheduleNetworkReResolve: a transient
// miss during a relay reconnect must not flip every effective
// URL back to the dead saved host. Keep the last-known route;
// we fall through to doConnect(url) and retry it with backoff.
_activeEndpoint.value = null
activeRelayEndpoint = resolved
}
if (targetUrl != null && normalizeRelayUrl(targetUrl) != url) {
Log.i(TAG, "scheduleReconnect: switching $url → ${normalizeRelayUrl(targetUrl)}")
connectToUrlOnMainPath(targetUrl)
connectToUrlOnMainPath(
targetUrl,
preserveReconnectBackoff = true,
)
} else {
doConnect(url)
}
@@ -0,0 +1,61 @@
package com.hermesandroid.relay.network.relay
/**
* Route-aware retry state for the Relay WebSocket.
*
* Automatic LAN/Tailscale fallback keeps the accumulated reconnect attempt so
* swapping URLs cannot restart exponential backoff. Socket-failure streaks are
* scoped to one URL, so failures on different roles cannot combine and poison
* the newly selected route.
*/
internal class RelayReconnectState {
@Volatile
var reconnectAttempt: Int = 0
private set
private var socketFailureRoute: String? = null
private var consecutiveSocketFailures: Int = 0
@Synchronized
fun beginExplicitConnect(route: String) {
reconnectAttempt = 0
resetSocketFailures(route)
}
@Synchronized
fun beginAutomaticRouteSwap(route: String) {
resetSocketFailures(route)
}
@Synchronized
fun nextReconnectAttempt(): Int {
reconnectAttempt++
return reconnectAttempt
}
@Synchronized
fun recordSocketFailure(route: String): Int {
if (socketFailureRoute != route) {
resetSocketFailures(route)
}
consecutiveSocketFailures++
return consecutiveSocketFailures
}
@Synchronized
fun connected(route: String) {
reconnectAttempt = 0
resetSocketFailures(route)
}
@Synchronized
fun reset() {
reconnectAttempt = 0
resetSocketFailures(null)
}
private fun resetSocketFailures(route: String?) {
socketFailureRoute = route
consecutiveSocketFailures = 0
}
}
@@ -98,6 +98,7 @@ class RelayVoiceClient(
private val webSocketFactory: ((Request, WebSocketListener) -> WebSocket)? = null,
private val realtimeResumeRetryIntervalMs: Long = REALTIME_RESUME_RETRY_INTERVAL_MS,
private val realtimeResumeRetryWindowMs: Long = REALTIME_RESUME_RETRY_WINDOW_MS,
private val voiceOutputFirstAudioTimeoutMs: Long = VOICE_OUTPUT_FIRST_AUDIO_TIMEOUT_MS,
) {
companion object {
@@ -108,6 +109,7 @@ class RelayVoiceClient(
private val WAV_AUDIO = "audio/wav".toMediaType()
private val OCTET_STREAM = "application/octet-stream".toMediaType()
private const val REALTIME_TIMEOUT_MS = 90_000L
private const val VOICE_OUTPUT_FIRST_AUDIO_TIMEOUT_MS = 15_000L
private const val REALTIME_AGENT_IDLE_TIMEOUT_MS = 90_000L
private const val REALTIME_AGENT_MAX_TURN_MS = 5 * 60_000L
private const val REALTIME_AGENT_WAIT_SLICE_MS = 1_000L
@@ -867,6 +869,7 @@ class RelayVoiceClient(
val resumeAttempted = AtomicBoolean(false)
val routeProbeRequested = AtomicBoolean(false)
val currentSocket = AtomicReference<WebSocket?>()
val firstAudioSeen = AtomicBoolean(false)
val socketGeneration = AtomicLong(0L)
val activeSocketGeneration = AtomicLong(0L)
val lastEventId = AtomicLong(0L)
@@ -980,6 +983,7 @@ class RelayVoiceClient(
}
onEvent(event)
if (event.isAudioDelta) {
firstAudioSeen.set(true)
event.audioEventId?.let {
lastPlayedAudioEventId.updateAndGet { current -> maxOf(current, it) }
}
@@ -1123,6 +1127,15 @@ class RelayVoiceClient(
onHandoff = onHandoff,
completeFailure = ::completeFailure,
)
val firstAudioWatchdog = launch {
delay(voiceOutputFirstAudioTimeoutMs)
if (!completed.get() && !firstAudioSeen.get()) {
completeFailure(
"Voice output produced no audio within ${voiceOutputFirstAudioTimeoutMs}ms",
)
currentSocket.get()?.cancel()
}
}
try {
withTimeout(REALTIME_TIMEOUT_MS) {
finished.await()
@@ -1132,6 +1145,7 @@ class RelayVoiceClient(
socket.close(1001, "timeout")
Result.failure(IOException("Voice output timed out", e))
} finally {
firstAudioWatchdog.cancel()
routeWatcher?.cancel()
}
}
@@ -1278,6 +1292,7 @@ class RelayVoiceClient(
model: String? = null,
voice: String? = null,
sampleRate: Int? = null,
finalAnswerOnly: Boolean = false,
onHandoff: (VoiceHandoffEvent) -> Unit = {},
turnInputs: kotlinx.coroutines.channels.ReceiveChannel<RealtimeTurnInput>? = null,
onTurnComplete: (RealtimeVoiceSummary) -> Unit = {},
@@ -1309,6 +1324,7 @@ class RelayVoiceClient(
model = model,
voice = voice,
sampleRate = sampleRate,
finalAnswerOnly = finalAnswerOnly,
)
if (sessionResult.isFailure) {
return@withContext Result.failure(sessionResult.exceptionOrNull() ?: IOException("Realtime agent session failed"))
@@ -1822,6 +1838,28 @@ class RelayVoiceClient(
if (event.type == "hermes.run.promoted") {
longRunningTurn.set(true)
Log.i(TAG, "Realtime agent turn marked long-running (run promoted); relaxing idle guard")
if (persistent &&
event.spokenHandoff == false &&
activeTurn.compareAndSet(true, false)
) {
Log.i(
TAG,
"Realtime agent foreground turn ended at silent background promotion",
)
onTurnComplete(
RealtimeVoiceSummary(
provider = event.provider ?: session.provider,
model = event.model ?: session.model,
voice = event.voice ?: session.voice,
sampleRate = session.sampleRate,
audioChunks = audioChunks,
audioBytes = audioBytes,
firstAudioMs = event.firstAudioMs,
responseDoneMs = event.responseDoneMs,
eventLogPath = event.eventLogPath ?: session.eventLogPath,
)
)
}
}
if (event.isAudioDelta) {
audioChunks += 1
@@ -1847,13 +1885,12 @@ class RelayVoiceClient(
responseDoneMs = event.responseDoneMs,
eventLogPath = event.eventLogPath ?: session.eventLogPath,
)
if (persistent) {
if (persistent && activeTurn.compareAndSet(true, false)) {
// Turn boundary, not session boundary: keep the socket
// open for the next utterance.
activeTurn.set(false)
longRunningTurn.set(false)
onTurnComplete(summary)
} else {
} else if (!persistent) {
if (claimTerminalSocket(
webSocket,
generation,
@@ -2679,6 +2716,7 @@ class RelayVoiceClient(
model: String? = null,
voice: String? = null,
sampleRate: Int? = null,
finalAnswerOnly: Boolean = false,
): Result<RealtimeSessionResponse> {
val body = buildJsonObject {
putProfile()
@@ -2694,6 +2732,9 @@ class RelayVoiceClient(
sampleRate?.takeIf { it > 0 }?.let {
put("sample_rate", JsonPrimitive(it))
}
if (finalAnswerOnly) {
put("final_answer_only", JsonPrimitive(true))
}
chatSessionId?.trim()?.takeIf { it.isNotBlank() }?.let {
put("chat_session_id", JsonPrimitive(it))
}
@@ -2980,6 +3021,9 @@ class RelayVoiceClient(
responseDoneMs = (metrics?.get("response_done_ms") as? JsonPrimitive)?.doubleOrNull,
tier = (obj["tier"] as? JsonPrimitive)?.contentOrNull,
floor = (obj["floor"] as? JsonPrimitive)?.contentOrNull,
spokenHandoff = (obj["spoken_handoff"] as? JsonPrimitive)
?.contentOrNull
?.toBooleanStrictOrNull(),
activeToolName = (obj["active_tool_name"] as? JsonPrimitive)?.contentOrNull,
completedToolCount = (obj["completed_tool_count"] as? JsonPrimitive)?.intOrNull
?: (obj["tool_count"] as? JsonPrimitive)?.intOrNull,
@@ -3374,6 +3418,7 @@ data class RealtimeVoiceEvent(
// ADR 33: background-run promotion fields.
val tier: String? = null,
val floor: String? = null,
val spokenHandoff: Boolean? = null,
// hermes.run.progress extras — drive the live background-run chip.
val activeToolName: String? = null,
val completedToolCount: Int? = null,
@@ -47,6 +47,16 @@ data class RouteProbeOutcome(
val atMillis: Long,
)
/**
* Service whose reachability is being resolved. Standard Hermes surfaces and
* Relay are intentionally independent: a healthy Dashboard must not vouch for
* a dead Relay listener on the same host.
*/
enum class EndpointSurface {
Standard,
Relay,
}
/**
* Picks the highest-priority **reachable** [EndpointCandidate] from a
* per-device list, driven by ADR 24 "Multi-endpoint pairing + network-aware
@@ -124,9 +134,14 @@ class EndpointResolver(
*/
val probeOutcomes: StateFlow<Map<String, RouteProbeOutcome>> = _probeOutcomes.asStateFlow()
private fun recordOutcome(candidate: EndpointCandidate, reachable: Boolean, detail: String?) {
private fun recordOutcome(
candidate: EndpointCandidate,
surface: EndpointSurface,
reachable: Boolean,
detail: String?,
) {
_probeOutcomes.update { outcomes ->
outcomes + (cacheKey(candidate) to RouteProbeOutcome(
outcomes + (cacheKey(candidate, surface) to RouteProbeOutcome(
reachable = reachable,
detail = detail,
atMillis = clock(),
@@ -167,21 +182,44 @@ class EndpointResolver(
private const val PROBE_TIMEOUT_DETAIL = "No answer (timed out)"
/**
* Stable cache key for a candidate: `"<role>|<primary host>:<port>"`.
* Stable cache key for one candidate surface:
* `"<surface>|<role>|<surface host>:<port>"`.
* Roles are preserved case-verbatim (HMAC canonicalization contract)
* but hostnames are lowercased — two roles pointing at the same
* host:port share reachability state.
*/
internal fun cacheKey(candidate: EndpointCandidate): String =
"${candidate.role}|${candidate.routeAuthority() ?: candidate.primaryRouteUrl().orEmpty().lowercase()}"
internal fun cacheKey(
candidate: EndpointCandidate,
surface: EndpointSurface = EndpointSurface.Standard,
): String {
val authority = when (surface) {
EndpointSurface.Standard ->
candidate.routeAuthority() ?: candidate.primaryRouteUrl().orEmpty().lowercase()
EndpointSurface.Relay ->
routeAuthority(candidate.relay?.url).orEmpty()
}
return "${surface.name.lowercase()}|${candidate.role}|$authority"
}
private fun routeAuthority(rawUrl: String?): String? {
val candidate = rawUrl?.trim()?.takeIf { it.isNotBlank() } ?: return null
val httpUrl = when {
candidate.startsWith("ws://", ignoreCase = true) ->
"http://${candidate.substringAfter("://")}"
candidate.startsWith("wss://", ignoreCase = true) ->
"https://${candidate.substringAfter("://")}"
else -> candidate
}
return httpUrl.toHttpUrlOrNull()?.let { url -> "${url.host}:${url.port}" }
}
}
/**
* Run the resolver against [candidates].
*
* 1. Group by `priority` ascending.
* 2. For each priority group, race a HEAD /health probe against every
* candidate in the group (2 s per candidate). First 2xx wins; ties
* 2. For each priority group, race the selected surface's health probe
* against every candidate in the group. First 2xx wins; ties
* broken by whichever response lands first.
* 3. If the entire group is unreachable, fall through to the next
* priority group.
@@ -193,37 +231,42 @@ class EndpointResolver(
* its tier). An empty [candidates] list returns null immediately without
* touching the network.
*/
suspend fun resolve(candidates: List<EndpointCandidate>): EndpointCandidate? {
if (candidates.isEmpty()) return null
suspend fun resolve(
candidates: List<EndpointCandidate>,
surface: EndpointSurface = EndpointSurface.Standard,
): EndpointCandidate? {
val eligible = candidates.filter { probeTarget(it, surface) != null }
if (eligible.isEmpty()) return null
// Strict priority: sort ascending so priority-0 lands first. Grouping
// preserves emitted order within a priority class (DNS SRV parity).
val groups = candidates.groupBy { it.priority }.toSortedMap()
val groups = eligible.groupBy { it.priority }.toSortedMap()
for ((priority, group) in groups) {
Log.d(TAG, "probing priority=$priority group (size=${group.size})")
val winner = raceGroup(group)
val winner = raceGroup(group, surface)
if (winner != null) {
val winnerUrl = probeTarget(winner, surface)?.baseUrl
Log.i(TAG, "resolve winner: role=${winner.role} " +
"route=${winner.primaryRouteUrl()} priority=$priority")
"surface=$surface route=$winnerUrl priority=$priority")
DiagnosticsLog.record(
category = DiagnosticCategory.Endpoint,
severity = DiagnosticSeverity.Info,
title = context?.getString(R.string.endpoint_diag_selected) ?: "Endpoint selected",
detail = "priority=$priority",
endpointRole = winner.role,
url = winner.primaryRouteUrl(),
url = winnerUrl,
)
return winner
}
}
Log.w(TAG, "resolve: no reachable candidate across ${candidates.size} record(s)")
Log.w(TAG, "resolve: no reachable $surface candidate across ${eligible.size} record(s)")
DiagnosticsLog.record(
category = DiagnosticCategory.Endpoint,
severity = DiagnosticSeverity.Warning,
title = context?.getString(R.string.endpoint_diag_no_reachable) ?: "No reachable endpoint",
detail = "${candidates.size} configured route(s) failed health probes",
detail = "${eligible.size} configured $surface route(s) failed health probes",
)
return null
}
@@ -237,17 +280,20 @@ class EndpointResolver(
* for "first 2xx wins" so latency matters. The losing probes' results
* still land in the cache, though, so the next call benefits.
*/
private suspend fun raceGroup(group: List<EndpointCandidate>): EndpointCandidate? {
private suspend fun raceGroup(
group: List<EndpointCandidate>,
surface: EndpointSurface,
): EndpointCandidate? {
if (group.isEmpty()) return null
if (group.size == 1) {
val only = group.first()
return if (isReachable(only)) only else null
return if (isReachable(only, surface)) only else null
}
// Fast-path: any cached-reachable candidate wins immediately without
// touching the network.
for (candidate in group) {
val cached = probeCache[cacheKey(candidate)]
val cached = probeCache[cacheKey(candidate, surface)]
if (cached != null && cached.expiresAt > clock() && cached.reachable) {
return candidate
}
@@ -256,7 +302,7 @@ class EndpointResolver(
return coroutineScope {
val deferred = group.map { candidate ->
async(Dispatchers.IO) {
if (isReachable(candidate)) candidate else null
if (isReachable(candidate, surface)) candidate else null
}
}
// Collect results in arrival order: iterate through awaitAll +
@@ -276,8 +322,11 @@ class EndpointResolver(
* [probeCache] first; on miss or expiry, runs a HEAD /health probe and
* records the result.
*/
private suspend fun isReachable(candidate: EndpointCandidate): Boolean {
val key = cacheKey(candidate)
private suspend fun isReachable(
candidate: EndpointCandidate,
surface: EndpointSurface,
): Boolean {
val key = cacheKey(candidate, surface)
val now = clock()
val cached = probeCache[key]
if (cached != null && cached.expiresAt > now) {
@@ -285,7 +334,7 @@ class EndpointResolver(
return cached.reachable
}
val reachable = probe(candidate)
val reachable = probe(candidate, surface)
val ttl = if (reachable) CACHE_TTL_MS else NEGATIVE_CACHE_TTL_MS
probeCache[key] = CacheEntry(expiresAt = now + ttl, reachable = reachable)
return reachable
@@ -299,9 +348,12 @@ class EndpointResolver(
* Returns false on any failure (timeout, I/O, non-2xx, invalid URL).
* We never raise: a bad record shouldn't crash the connect loop.
*/
private suspend fun probe(candidate: EndpointCandidate): Boolean {
private suspend fun probe(
candidate: EndpointCandidate,
surface: EndpointSurface,
): Boolean {
val startedAtMs = clock()
val target = probeTarget(candidate)
val target = probeTarget(candidate, surface)
val url = target?.requestUrl?.toHttpUrlOrNull()
?: run {
Log.w(TAG, "probe: invalid url for role=${candidate.role}")
@@ -313,7 +365,7 @@ class EndpointResolver(
endpointRole = candidate.role,
url = candidate.primaryRouteUrl(),
)
recordOutcome(candidate, reachable = false, detail = "Invalid route URL")
recordOutcome(candidate, surface, reachable = false, detail = "Invalid route URL")
return false
}
val fastClient = httpClient.newBuilder()
@@ -351,6 +403,7 @@ class EndpointResolver(
)
recordOutcome(
candidate,
surface,
reachable = ok,
detail = if (ok) null else "HTTP ${resp.code} from ${target.path}",
)
@@ -366,7 +419,7 @@ class EndpointResolver(
url = target.baseUrl,
elapsedMs = clock() - startedAtMs,
)
recordOutcome(candidate, reachable = false, detail = PROBE_TIMEOUT_DETAIL)
recordOutcome(candidate, surface, reachable = false, detail = PROBE_TIMEOUT_DETAIL)
false
}
} catch (_: TimeoutCancellationException) {
@@ -379,7 +432,7 @@ class EndpointResolver(
url = target.baseUrl,
elapsedMs = clock() - startedAtMs,
)
recordOutcome(candidate, reachable = false, detail = PROBE_TIMEOUT_DETAIL)
recordOutcome(candidate, surface, reachable = false, detail = PROBE_TIMEOUT_DETAIL)
false
} catch (e: Exception) {
Log.d(TAG, "probe failed role=${candidate.role} " +
@@ -393,14 +446,20 @@ class EndpointResolver(
url = target.baseUrl,
elapsedMs = clock() - startedAtMs,
)
recordOutcome(candidate, reachable = false, detail = humanProbeFailure(e))
recordOutcome(candidate, surface, reachable = false, detail = humanProbeFailure(e))
false
}
}
}
/** Choose the standard Dashboard/Gateway surface first when advertised. */
private fun probeTarget(candidate: EndpointCandidate): ProbeTarget? {
private fun probeTarget(
candidate: EndpointCandidate,
surface: EndpointSurface,
): ProbeTarget? {
if (surface == EndpointSurface.Relay) {
return relayProbeTarget(candidate)
}
candidate.dashboard?.url
?.trim()
?.trimEnd('/')
@@ -421,6 +480,10 @@ class EndpointResolver(
)
}
return relayProbeTarget(candidate)
}
private fun relayProbeTarget(candidate: EndpointCandidate): ProbeTarget? {
candidate.relay?.url
?.trim()
?.trimEnd('/')
@@ -467,13 +530,21 @@ class EndpointResolver(
* transition can skip the known-dead active route without suppressing a
* valid fallback for the whole positive cache window.
*/
fun markUnreachable(candidate: EndpointCandidate) {
val key = cacheKey(candidate)
fun markUnreachable(
candidate: EndpointCandidate,
surface: EndpointSurface = EndpointSurface.Standard,
) {
val key = cacheKey(candidate, surface)
probeCache[key] = CacheEntry(
expiresAt = clock() + NEGATIVE_CACHE_TTL_MS,
reachable = false,
)
recordOutcome(candidate, reachable = false, detail = "Network changed — assumed offline")
recordOutcome(
candidate,
surface,
reachable = false,
detail = "Network changed — assumed offline",
)
}
/**
@@ -65,13 +65,14 @@ object ProfileApiUrlResolver {
val dedicated = resolveForConnection(profileApiUrl, base)
if (dedicated != null) return dedicated
val profile = selectedProfileName
?.trim()
?.takeIf { it.isNotBlank() && !it.equals("default", ignoreCase = true) }
?: return base
val isKnownMultiplexProfile = gatewayMode.equals("multiplex", ignoreCase = true) &&
servedProfiles.any { it.equals(profile, ignoreCase = false) }
if (!isKnownMultiplexProfile) return base
val profile = selectedProfileName?.trim() ?: return base
if (!usesMultiplexProfileKey(
profileApiUrl = profileApiUrl,
selectedProfileName = profile,
gatewayMode = gatewayMode,
servedProfiles = servedProfiles,
)
) return base
val root = base?.toHttpUrlOrNull() ?: return base
return root.newBuilder()
@@ -82,6 +83,27 @@ object ProfileApiUrlResolver {
.trimEnd('/')
}
/**
* A profile-specific credential is required only for the positively
* identified shared `/p/<profile>` mirror. Dedicated profile APIs retain
* the connection credential contract, while default/legacy/unknown routes
* stay on the root client.
*/
fun usesMultiplexProfileKey(
profileApiUrl: String?,
selectedProfileName: String?,
gatewayMode: String?,
servedProfiles: Collection<String>,
): Boolean {
if (normalize(profileApiUrl) != null) return false
val profile = selectedProfileName
?.trim()
?.takeIf { it.isNotBlank() && !it.equals("default", ignoreCase = true) }
?: return false
return gatewayMode.equals("multiplex", ignoreCase = true) &&
servedProfiles.any { it == profile }
}
private fun isLocalBindHost(host: String): Boolean {
return when (host.lowercase().trim('[', ']')) {
"localhost", "127.0.0.1", "0.0.0.0", "::1", "::" -> true
@@ -320,47 +320,54 @@ class ChatHandler {
fun boolField(name: String): Boolean? = (payload[name] as? JsonPrimitive)?.booleanOrNull
val toolName = textField("tool_name", "tool", "name") ?: "unknown"
val callId = textField("call_id", "tool_call_id") ?: toolName
// The caller owns one client placeholder id for the whole Relay stream.
// message.started can replace that domain id with the server id while
// uiKey deliberately retains the client id. Resolve the current domain
// id before every event so the tail keeps mutating the same visible row.
val currentMessageId = _messages.value.findLast { message ->
message.id == messageId || message.uiKey == messageId
}?.id ?: messageId
when (envelope.event) {
"message.started" -> {
val msgObj = payload["message"] as? JsonObject
val serverMsgId = (msgObj?.get("id") as? JsonPrimitive)?.contentOrNull
if (!serverMsgId.isNullOrBlank()) replaceMessageId(messageId, serverMsgId)
if (!serverMsgId.isNullOrBlank()) replaceMessageId(currentMessageId, serverMsgId)
}
"assistant.delta" -> {
textField("delta", "content", "text")?.let { onTextDelta(messageId, it) }
textField("delta", "content", "text")?.let { onTextDelta(currentMessageId, it) }
}
"tool.progress" -> {
textField("delta", "thinking_delta", "thinking", "text", "message")?.let {
onThinkingDelta(messageId, it)
onThinkingDelta(currentMessageId, it)
}
}
"tool.pending", "tool.started" -> onToolCallStart(messageId, callId, toolName)
"tool.completed" -> onToolCallComplete(messageId, callId, textField("result_preview", "summary", "message"))
"tool.failed" -> onToolCallFailed(messageId, callId, textField("error", "message") ?: "Tool failed")
"tool.pending", "tool.started" -> onToolCallStart(currentMessageId, callId, toolName)
"tool.completed" -> onToolCallComplete(currentMessageId, callId, textField("result_preview", "summary", "message"))
"tool.failed" -> onToolCallFailed(currentMessageId, callId, textField("error", "message") ?: "Tool failed")
"memory.updated", "skill.loaded" -> {
val label = when (envelope.event) {
"memory.updated" -> "Memory"
else -> "Skill"
}
addMessageBadges(messageId, listOf(label))
addMessageBadges(currentMessageId, listOf(label))
}
"artifact.created" -> {
addMessageBadges(messageId, listOf("Artifact"))
addMessageBadges(currentMessageId, listOf("Artifact"))
textField("url", "path", "preview", "title")?.takeIf { it.isNotBlank() }?.let {
onThinkingDelta(messageId, "Artifact: $it")
onThinkingDelta(currentMessageId, "Artifact: $it")
}
}
"assistant.completed" -> {
if (boolField("interrupted") == true) {
onStreamError("Response interrupted")
} else {
onTurnComplete(messageId)
onTurnComplete(currentMessageId)
}
}
"run.completed", "done" -> onStreamComplete(messageId)
"run.completed", "done" -> onStreamComplete(currentMessageId)
"error" -> {
addMessageBadges(messageId, listOf("Error"))
addMessageBadges(currentMessageId, listOf("Error"))
onStreamError(textField("message", "error") ?: "Unknown error")
}
"session.created", "run.started" -> Unit
@@ -372,6 +379,7 @@ class ChatHandler {
fun addUserMessage(message: ChatMessage) {
_messages.update { list ->
if (list.any { it.uiKey == message.uiKey }) return@update list
(list + message).let { if (it.size > MAX_MESSAGES) it.drop(it.size - MAX_MESSAGES) else it }
}
}
@@ -942,6 +950,7 @@ class ChatHandler {
fun addPlaceholderMessage(message: ChatMessage) {
_isStreaming.value = true
_messages.update { list ->
if (list.any { it.uiKey == message.uiKey }) return@update list
(list + message).let { if (it.size > MAX_MESSAGES) it.drop(it.size - MAX_MESSAGES) else it }
}
}
@@ -957,6 +966,7 @@ class ChatHandler {
fun restoreInFlightTurn(
checkpoint: ChatTurnCheckpoint,
upstreamAssistantText: String? = null,
corrections: List<String> = emptyList(),
) {
val user = checkpoint.user
val assistant = checkpoint.assistant
@@ -1085,13 +1095,53 @@ class ChatHandler {
timestamp = user.timestamp,
)
}
val insertBeforeAsk = withUser.indexOfFirst {
// Newer gateways retain the original prompt in inflight.user and
// expose every accepted active-turn redirect separately. Restore
// those corrections as ordinary user bubbles before the assistant
// row. Consume matching already-present rows first so repeated
// resume/checkpoint passes cannot duplicate them.
val originalUserIndex = withUser.indexOfFirst { it.id == user.id }
.takeIf { it >= 0 }
?: withUser.indexOfFirst {
it.role == MessageRole.USER && it.content.trim() == user.content.trim()
}
val existingAfterOriginal = withUser
.drop((originalUserIndex + 1).coerceAtLeast(0))
.filter { it.role == MessageRole.USER }
.map { it.content.trim() }
.toMutableList()
val restoredCorrections = corrections
.map { it.trim() }
.filter { it.isNotBlank() }
.mapIndexedNotNull { index, correction ->
val existingIndex = existingAfterOriginal.indexOf(correction)
if (existingIndex >= 0) {
existingAfterOriginal.removeAt(existingIndex)
null
} else {
ChatMessage(
id = "${user.id}-correction-${index + 1}",
role = MessageRole.USER,
content = correction,
timestamp = user.timestamp + index + 1L,
)
}
}
val firstAskIndex = withUser.indexOfFirst {
it.clientOnly && it.id.startsWith("ask-")
}
val withCorrections = if (firstAskIndex >= 0) {
withUser.toMutableList().apply { addAll(firstAskIndex, restoredCorrections) }
} else {
withUser + restoredCorrections
}
val insertBeforeAsk = withCorrections.indexOfFirst {
it.clientOnly && it.id.startsWith("ask-")
}
val restored = if (insertBeforeAsk >= 0) {
withUser.toMutableList().apply { add(insertBeforeAsk, restoredAssistant) }
withCorrections.toMutableList().apply { add(insertBeforeAsk, restoredAssistant) }
} else {
withUser + restoredAssistant
withCorrections + restoredAssistant
}
restored.let { list ->
if (list.size > MAX_MESSAGES) list.drop(list.size - MAX_MESSAGES) else list
@@ -1223,6 +1273,14 @@ class ChatHandler {
// so we can attach results back to the originating assistant message's ToolCall
val toolResults = items.filter { it.role == "tool" }
.associateBy { it.toolCallId }
// A reconnect/rejoin history response can repeat a persisted message row.
// Chat's LazyColumn renders domain ids as stable keys (via ChatMessage.uiKey),
// so allowing both copies through would crash Compose before either copy
// could be reconciled. A domain id identifies one persisted message: retain
// its first transcript position while adopting the latest repeated snapshot.
// Rows without ids remain independent, and tool/hidden rows keep their
// separate handling above/below.
val renderedItems = coalesceRenderedHistoryItems(items)
// Accumulator for media markers we find in loaded content — fired AFTER
// the wholesale `_messages.value = ...` assignment so the ViewModel's
@@ -1240,8 +1298,8 @@ class ChatHandler {
// silently misses those rows, so a gateway turn's tokens/badges survived
// only if a content match happened to cover them. See
// [reconcileLiveIdsToServer].
val serverItemIds = items.mapNotNullTo(HashSet()) { it.id }
val idRemap = reconcileLiveIdsToServer(items, serverItemIds)
val serverItemIds = renderedItems.mapNotNullTo(HashSet()) { it.id }
val idRemap = reconcileLiveIdsToServer(renderedItems, serverItemIds)
// Carry CLIENT-ONLY enrichment forward across the reload, keyed by the
// RECONCILED message id. The server transcript (MessageItem) rebuilds
@@ -1278,7 +1336,7 @@ class ChatHandler {
// clientOnly bubbles (same exchange, pre-sync copy).
val syncedRealtimeTurnContents = mutableSetOf<String>()
val loaded = items.mapNotNull { item ->
val loaded = renderedItems.mapNotNull { item ->
val displayKind = item.displayKind?.trim()?.lowercase()
if (displayKind == "hidden") return@mapNotNull null
val role = when {
@@ -1540,6 +1598,34 @@ class ChatHandler {
}
}
/**
* Collapse replayed visible history rows by their authoritative message id.
*
* Replacing the value at its first-seen slot preserves transcript ordering;
* the last repeated value wins so a later, more complete snapshot is not lost.
* Null ids cannot be proven identical and therefore remain separate rows.
*/
private fun coalesceRenderedHistoryItems(items: List<MessageItem>): List<MessageItem> {
val firstSlotById = HashMap<String, Int>()
val coalesced = ArrayList<MessageItem>(items.size)
for (item in items) {
if (renderedRoleOf(item) == null) continue
val id = item.id
if (id == null) {
coalesced += item
continue
}
val existingSlot = firstSlotById[id]
if (existingSlot == null) {
firstSlotById[id] = coalesced.size
coalesced += item
} else {
coalesced[existingSlot] = item
}
}
return coalesced
}
/** One adoptable server row during id reconciliation. `taken` enforces consume-once. */
private class ReconcileSlot(
val serverId: String,
@@ -365,8 +365,10 @@ class DashboardApiClient(
* `available=false` with an empty list when the server has no API key
* configured; the API key itself never leaves the server.
*/
suspend fun getElevenLabsVoices(): Result<ElevenLabsVoices> = withContext(Dispatchers.IO) {
getJson("/api/audio/elevenlabs/voices").mapCatching { parseElevenLabsVoices(it) }
suspend fun getElevenLabsVoices(profile: String? = null): Result<ElevenLabsVoices> =
withContext(Dispatchers.IO) {
getJson("/api/audio/elevenlabs/voices${profileQuery(profile)}")
.mapCatching { parseElevenLabsVoices(it) }
}
/**
@@ -1058,7 +1060,12 @@ class DashboardApiClient(
}
}
fun gatewayWebSocketUrl(baseUrl: String, ticket: String, path: String = "/api/ws"): String? {
fun gatewayWebSocketUrl(
baseUrl: String,
ticket: String,
path: String = "/api/ws",
profile: String? = null,
): String? {
val httpUrl = baseUrl.trim().trimEnd('/').toHttpUrlOrNull() ?: return null
val websocketPrefix = when (httpUrl.scheme) {
"https" -> "wss://"
@@ -1074,6 +1081,11 @@ class DashboardApiClient(
val url = httpUrl.newBuilder()
.encodedPath(encodedPath)
.addQueryParameter("ticket", ticket)
.apply {
profile?.trim()?.takeIf { it.isNotBlank() }?.let {
addQueryParameter("profile", it)
}
}
.build()
.toString()
return websocketPrefix + url.substringAfter("://")
@@ -31,6 +31,7 @@ import kotlinx.serialization.json.decodeFromJsonElement
import kotlinx.serialization.json.intOrNull
import kotlinx.serialization.json.longOrNull
import kotlinx.serialization.json.put
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
@@ -501,6 +502,10 @@ class GatewayChatClient(
* into the session's USER messages (counted from the first user
* message). The server drops that message and everything after it
* before running [text] as a fresh turn.
* @param queuedFollowUp true only when Android is draining a prompt the
* user explicitly queued behind an active turn. Newer gateways use the
* additive `queued:true` marker to preserve run-after semantics while
* the previous turn is still settling; older gateways ignore it.
* @param onPreflightFailure invoked INSTEAD of starting the turn when the
* gateway could not be reached / authenticated / the prompt could not
* be submitted — i.e. nothing started server-side, so the caller can
@@ -514,6 +519,7 @@ class GatewayChatClient(
callbacks: GatewayTurnCallbacks,
attachments: List<GatewayAttachment> = emptyList(),
truncateBeforeUserOrdinal: Int? = null,
queuedFollowUp: Boolean = false,
onPreflightFailure: (reason: String) -> Unit,
): ActiveTurnHandle {
val turn = GatewayTurn(dispatchOn(callbacks))
@@ -550,6 +556,7 @@ class GatewayChatClient(
put("session_id", liveSessionId ?: error("no live session"))
put("text", text)
truncateBeforeUserOrdinal?.let { put("truncate_before_user_ordinal", it) }
if (queuedFollowUp) put("queued", true)
},
// Long-running RPC, not a generic 15s ack — see the
// constant's doc. The idle watchdog (armed above, reset by
@@ -573,8 +580,22 @@ class GatewayChatClient(
}
if (activeTurn === turn) activeTurn = null
turn.disarmWatchdog()
val submitError = submitted.exceptionOrNull()
if ((submitError as? GatewayRpcException)?.code == ACTIVE_SESSION_CAP_REJECTION) {
// Authoritative policy rejection: the gateway received
// the prompt and deliberately refused to create the
// first turn. Falling back to SSE would bypass the cap
// and duplicate the optimistic user turn on another
// transport. Surface the holder-aware upstream message
// through the normal failed-turn callback instead.
turn.tracer.done("submit-rejected")
turn.callbacks.onError(
submitError.message ?: "Hermes rejected the new session",
)
return@launch
}
throw GatewayPreflightException(
submitted.exceptionOrNull()?.message ?: "prompt.submit failed",
submitError?.message ?: "prompt.submit failed",
)
}
turn.tracer.mark("submit")
@@ -881,6 +902,16 @@ class GatewayChatClient(
user = value.stringField("user").orEmpty(),
assistant = value.stringField("assistant").orEmpty(),
streaming = value.booleanField("streaming") == true,
corrections = (value["corrections"] as? JsonArray)
?.mapNotNull { correction ->
(correction as? JsonPrimitive)
?.contentOrNull
?.trim()
?.takeIf { it.isNotBlank() }
?.take(MAX_RECOVERED_CORRECTION_CHARS)
}
?.take(MAX_RECOVERED_CORRECTIONS)
.orEmpty(),
status = value.stringField("status"),
error = value.stringField("error"),
recoverable = value.booleanField("recoverable") == true,
@@ -1159,6 +1190,59 @@ class GatewayChatClient(
.onSuccess { commandsCatalogCache = it }
}
/**
* Fetch the upstream gateway's cropped preview for a Petdex pet.
*
* A missing thumbnail is represented by a successful `null`, matching the
* gateway's fail-open `{ "ok": false }` response. RPC errors, including
* method-not-found on older upstream gateways, remain failures so callers
* can distinguish an unavailable capability from a missing image.
*/
suspend fun petThumbnail(
slug: String,
spritesheetUrl: String? = null,
profile: String? = currentSessionProfile(),
): Result<String?> {
val normalizedSlug = slug.trim()
if (!PETDEX_SLUG.matches(normalizedSlug)) {
return Result.failure(IllegalArgumentException("invalid Petdex slug"))
}
val normalizedUrl = spritesheetUrl?.trim()?.takeIf { it.isNotEmpty() }
if (normalizedUrl != null && !isTrustedPetdexAssetUrl(normalizedUrl)) {
return Result.failure(IllegalArgumentException("invalid Petdex spritesheet URL"))
}
try {
connectMutex.withLock { ensureConnected() }
} catch (e: Exception) {
return Result.failure(e)
}
return rpc(
"pet.thumb",
buildJsonObject {
put("slug", normalizedSlug)
normalizedUrl?.let { put("url", it) }
profile?.trim()?.takeIf { it.isNotEmpty() }?.let { put("profile", it) }
},
).mapCatching { response ->
val ok = (response["ok"] as? JsonPrimitive)?.booleanOrNull
?: throw GatewayRpcException("pet.thumb returned an invalid response")
val responseSlug = (response["slug"] as? JsonPrimitive)?.contentOrNull
if (responseSlug != normalizedSlug) {
throw GatewayRpcException("pet.thumb returned a mismatched slug")
}
if (!ok) return@mapCatching null
val dataUri = (response["dataUri"] as? JsonPrimitive)?.contentOrNull
if (dataUri == null || !isValidPetThumbnailDataUri(dataUri)) {
throw GatewayRpcException("pet.thumb returned an invalid thumbnail")
}
dataUri
}
}
/**
* Fetch the current chat session's running and recently-finished background
* processes. Callers never provide a session id: this wrapper resolves and
@@ -1270,6 +1354,22 @@ class GatewayChatClient(
}
}
/**
* List personalities through upstream's slash completer. Unlike the
* dashboard config schema, this resolves the CLI config that contains both
* built-in and profile-defined personalities, matching `/personality` in
* the desktop and TUI.
*/
suspend fun personalityOptions(): Result<List<String>> {
if (webSocket == null || readySignal?.isCompleted != true) {
return Result.failure(GatewayRpcException("not connected"))
}
return rpc(
"complete.slash",
buildJsonObject { put("text", "/personality ") },
).map(::parseGatewayPersonalityOptions)
}
/**
* Set the personality the way the desktop + TUI do (`config.set
* {key:"personality"}`). The gateway persists `display.personality` +
@@ -2895,6 +2995,22 @@ class GatewayChatClient(
)
}
internal fun parseGatewayPersonalityOptions(result: JsonObject): List<String> =
(result["items"] as? JsonArray)
.orEmpty()
.mapNotNull { item ->
(item as? JsonObject)
?.stringField("text")
?.trim()
?.removePrefix("/personality")
?.trim()
?.takeIf {
it.isNotBlank() &&
it.lowercase() !in setOf("none", "default", "neutral")
}
}
.distinctBy { it.lowercase() }
/** Outcome of an active-turn correction — Rejected and Failed both mean "queue locally instead". */
enum class SteerResult {
/** Server accepted the active-turn correction. */
@@ -2931,6 +3047,31 @@ internal class GatewayConnectAttemptException(message: String) : Exception(messa
internal class GatewayRpcException(message: String, val code: Int? = null) : Exception(message)
private const val JSONRPC_METHOD_NOT_FOUND = -32601
private const val ACTIVE_SESSION_CAP_REJECTION = 4090
private const val MAX_RECOVERED_CORRECTIONS = 32
private const val MAX_RECOVERED_CORRECTION_CHARS = 32_768
private const val PET_THUMB_DATA_PREFIX = "data:image/png;base64,"
private const val MAX_PET_THUMB_BASE64_CHARS = 512 * 1024
private val PETDEX_SLUG = Regex("[a-z0-9][a-z0-9-]{0,127}")
private val STANDARD_BASE64 = Regex("[A-Za-z0-9+/]*={0,2}")
private fun isTrustedPetdexAssetUrl(raw: String): Boolean {
val url = raw.toHttpUrlOrNull() ?: return false
return url.scheme == "https" &&
url.host == "assets.petdex.dev" &&
url.port == 443 &&
url.username.isEmpty() &&
url.password.isEmpty()
}
private fun isValidPetThumbnailDataUri(raw: String): Boolean {
if (!raw.startsWith(PET_THUMB_DATA_PREFIX)) return false
val payload = raw.substring(PET_THUMB_DATA_PREFIX.length)
return payload.isNotEmpty() &&
payload.length <= MAX_PET_THUMB_BASE64_CHARS &&
payload.length % 4 == 0 &&
STANDARD_BASE64.matches(payload)
}
data class GatewayCompressResult(
val status: String,
@@ -122,6 +122,8 @@ data class GatewayInflightTurn(
val user: String,
val assistant: String,
val streaming: Boolean,
/** Accepted active-turn redirects in display order; additive on newer Hermes. */
val corrections: List<String> = emptyList(),
val status: String? = null,
val error: String? = null,
val recoverable: Boolean = false,
@@ -1000,9 +1000,7 @@ class HermesApiClient(
// Personalities: config.agent.personalities { name: "system prompt", ... }
val agent = config["agent"] as? JsonObject
val personalitiesObj = agent?.get("personalities") as? JsonObject
val prompts = personalitiesObj?.entries?.associate { (key, value) ->
key to ((value as? kotlinx.serialization.json.JsonPrimitive)?.content ?: "")
} ?: emptyMap()
val prompts = parsePersonalityPrompts(personalitiesObj)
// Default display identity. Upstream Hermes currently uses
// config.display.personality for the active persona and often
@@ -108,13 +108,18 @@ class NativeDashboardAuthClient(
provider: String? = null,
): NativeDashboardAuthorization {
requireStrictLoopbackRedirect(redirectUri)
val verifier = randomBytes(32).base64Url()
// RFC 7636 uses unpadded Base64URL. Okio's base64Url() preserves
// trailing "=", which makes Hermes' standards-compliant S256
// comparison fail even though both sides hashed the same bytes.
val verifier = randomBytes(32).base64Url().trimEnd('=')
val challenge = MessageDigest.getInstance("SHA-256")
.digest(verifier.toByteArray(Charsets.US_ASCII))
.toByteString()
.base64Url()
.trimEnd('=')
val state = randomBytes(24).base64Url()
val root = "$baseUrl/auth/native/authorize".toHttpUrlOrNull()
val authorizationBaseUrl = resolveAuthorizationBaseUrl(provider)
val root = "$authorizationBaseUrl/auth/native/authorize".toHttpUrlOrNull()
?: throw IOException("Dashboard URL is not a valid http(s) address")
val url = root.newBuilder()
.addQueryParameter("code_challenge", challenge)
@@ -130,6 +135,41 @@ class NativeDashboardAuthClient(
return NativeDashboardAuthorization(url, verifier, state, generation)
}
/**
* A private-route dashboard may be configured with a canonical HTTPS
* callback origin for its provider. Starting the browser on the private
* origin would scope Hermes' temporary PKCE cookie to the wrong host, so
* discover the provider's declared callback and start native auth there.
* Token exchange still uses [baseUrl], keeping the resulting bearer bound
* to the active connection route.
*/
private fun resolveAuthorizationBaseUrl(provider: String?): String {
val configured = baseUrl.toHttpUrlOrNull() ?: return baseUrl
if (
!provider.equals("nous", ignoreCase = true) ||
configured.scheme != "http" ||
!isPrivateNetworkLiteral(configured.host)
) {
return baseUrl
}
val loginUrl = configured.newBuilder()
.addPathSegments("auth/login")
.addQueryParameter("provider", provider)
.addQueryParameter("next", "/")
.build()
val discoveryClient = client.newBuilder()
.followRedirects(false)
.followSslRedirects(false)
.build()
val location = discoveryClient.newCall(
Request.Builder().url(loginUrl).get().build(),
).execute().use { response ->
if (response.code !in 300..399) null else response.header("Location")
}
return canonicalDashboardBaseFromNousRedirect(location)
?: throw IOException("Dashboard did not advertise a secure Nous callback origin")
}
fun exchangeCallback(
authorization: NativeDashboardAuthorization,
callbackTarget: String,
@@ -280,7 +320,52 @@ internal class NativeDashboardCallbackException(
internal fun isNativeDashboardTransportEligible(baseUrl: String): Boolean {
val url = baseUrl.trim().trimEnd('/').toHttpUrlOrNull() ?: return false
return url.scheme == "https" ||
(url.scheme == "http" && url.host == "127.0.0.1")
(
url.scheme == "http" &&
(url.host == "127.0.0.1" || isPrivateNetworkLiteral(url.host))
)
}
/**
* Hermes already permits explicitly configured HTTP dashboard sessions on
* local routes. The brokered flow is no less protected than that cookie flow,
* but remains unavailable to arbitrary cleartext Internet hosts.
*/
private fun isPrivateNetworkLiteral(host: String): Boolean {
val octets = host.split('.').mapNotNull(String::toIntOrNull)
if (octets.size != 4 || octets.any { it !in 0..255 }) return false
val first = octets[0]
val second = octets[1]
return first == 10 ||
(first == 172 && second in 16..31) ||
(first == 192 && second == 168) ||
(first == 100 && second in 64..127)
}
internal fun canonicalDashboardBaseFromNousRedirect(location: String?): String? {
val providerUrl = location?.toHttpUrlOrNull() ?: return null
if (
providerUrl.scheme != "https" ||
!providerUrl.host.equals("portal.nousresearch.com", ignoreCase = true)
) {
return null
}
val callback = providerUrl.queryParameter("redirect_uri")
?.toHttpUrlOrNull()
?: return null
if (callback.scheme != "https") return null
val callbackSuffix = "/auth/callback"
if (!callback.encodedPath.endsWith(callbackSuffix)) return null
val basePath = callback.encodedPath
.removeSuffix(callbackSuffix)
.ifBlank { "/" }
return callback.newBuilder()
.encodedPath(basePath)
.query(null)
.fragment(null)
.build()
.toString()
.trimEnd('/')
}
/**
@@ -33,6 +33,24 @@ internal fun dashboardRedirectAuthMode(authFlows: List<String>): DashboardRedire
DashboardRedirectAuthMode.WebView
}
/**
* Nous Portal uses Cloudflare Turnstile and does not support embedded Android
* WebViews. Keep self-hosted OIDC on the dashboard cookie flow, but use the
* gateway's brokered system-browser flow for Nous when it is advertised.
*/
internal fun androidDashboardRedirectAuthMode(
providerName: String,
authFlows: List<String>,
): DashboardRedirectAuthMode =
if (
providerName.equals("nous", ignoreCase = true) &&
dashboardRedirectAuthMode(authFlows) == DashboardRedirectAuthMode.NativePkce
) {
DashboardRedirectAuthMode.NativePkce
} else {
DashboardRedirectAuthMode.WebView
}
/**
* Owns one native dashboard sign-in attempt.
*
@@ -0,0 +1,31 @@
package com.hermesandroid.relay.network.upstream
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
/**
* Parse both upstream personality formats: the original prompt string and the
* structured form introduced for richer descriptions, tone, and style.
*/
internal fun parsePersonalityPrompts(personalities: JsonObject?): Map<String, String> =
personalities
?.mapValues { (_, value) -> personalityPrompt(value) }
.orEmpty()
private fun personalityPrompt(value: JsonElement): String = when (value) {
is JsonPrimitive -> value.contentOrNull.orEmpty()
is JsonObject -> listOfNotNull(
value.stringValue("system_prompt"),
value.stringValue("tone")?.let { "Tone: $it" },
value.stringValue("style")?.let { "Style: $it" },
).joinToString("\n")
else -> ""
}
private fun JsonObject.stringValue(key: String): String? =
(this[key] as? JsonPrimitive)
?.contentOrNull
?.trim()
?.takeIf { it.isNotBlank() }
@@ -21,16 +21,27 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import okhttp3.WebSocket
import okhttp3.WebSocketListener
import okio.BufferedSink
import okio.ByteString
import java.io.File
import java.io.IOException
import java.io.OutputStream
import java.util.Base64
import java.util.concurrent.TimeUnit
internal fun standardHermesAudioUrl(
baseUrl: String,
path: String,
profile: String?,
) = "$baseUrl$path".toHttpUrlOrNull()?.newBuilder()?.apply {
profile?.trim()?.takeIf { it.isNotBlank() }?.let { addQueryParameter("profile", it) }
}?.build()
/**
* Standard (no-plugin) voice client — speaks the upstream **dashboard web
* server** contract that hermes-desktop's voice mode uses:
@@ -51,11 +62,9 @@ class StandardHermesVoiceClient(
private val context: Context,
private val dashboardHttpClientProvider: (String) -> OkHttpClient,
private val dashboardUrlProvider: () -> String?,
// Active chat profile name (null = default/launch). Sent DEFENSIVELY on
// /api/audio/speak: upstream `TTSSpeakRequest` is text-only and Pydantic
// ignores extra fields, so this is harmless today and forward-compatible if
// upstream ever adds profile-aware TTS. Until then, standard voice remains
// the host's global TTS (see VoiceViewModel's standard-voice profile notice).
// Active chat profile name (null = default/launch). Current upstream audio
// routes accept it as a query parameter so TTS, STT, streaming speech, and
// provider catalogs resolve through the same Hermes home as chat.
private val profileProvider: () -> String? = { null },
private val webSocketFactory: ((Request, WebSocketListener) -> WebSocket)? = null,
private val json: Json = Json {
@@ -86,17 +95,18 @@ class StandardHermesVoiceClient(
// malformed dashboard URL (a non-address pasted into that field, #131),
// and this runs before executeJson()'s try/catch, so the throw would
// escape withContext(IO) onto the calling coroutine and crash the app.
val httpUrl = "$baseUrl/api/audio/transcribe".toHttpUrlOrNull()
val httpUrl = dashboardAudioUrl(baseUrl, "/api/audio/transcribe")
?: return@withContext Result.failure(IOException("Hermes dashboard URL is not a valid address: $baseUrl"))
val dataUrl = buildAudioDataUrl(audioFile)
val payload = buildJsonObject {
put("data_url", dataUrl)
put("mime_type", mediaTypeForAudioFile(audioFile))
}
val mimeType = mediaTypeForAudioFile(audioFile)
val request = Request.Builder()
.url(httpUrl)
.post(json.encodeToString(JsonObject.serializer(), payload).toRequestBody(JSON_MEDIA))
// Stream the file through Base64 directly into OkHttp's sink. Building
// a JsonObject first retained the file bytes, a Base64 String, the JSON
// serializer's growing char buffer, and the final request bytes at the
// same time. A near-limit recording could therefore exhaust Android's
// 256 MB heap before the request reached the network (#271).
.post(standardHermesTranscriptionRequestBody(audioFile, mimeType))
.header("Accept", "application/json")
.build()
@@ -122,14 +132,11 @@ class StandardHermesVoiceClient(
// See transcribe(): guard the throwing url(String) so a malformed
// dashboard URL is a clean Result.failure, never a Main-thread crash.
val httpUrl = "$baseUrl/api/audio/speak".toHttpUrlOrNull()
val httpUrl = dashboardAudioUrl(baseUrl, "/api/audio/speak")
?: return@withContext Result.failure(IOException("Hermes dashboard URL is not a valid address: $baseUrl"))
val payload = buildJsonObject {
put("text", cleanText)
// Defensive only — upstream /api/audio/speak ignores it (text-only
// TTSSpeakRequest). Omitted for the default profile.
profileProvider()?.trim()?.takeIf { it.isNotBlank() }?.let { put("profile", it) }
}
val request = Request.Builder()
.url(httpUrl)
@@ -176,6 +183,7 @@ class StandardHermesVoiceClient(
baseUrl = baseUrl,
ticket = ticket,
path = "/api/audio/speak-stream",
profile = activeProfile(),
) ?: throw IOException("Could not build Hermes speech stream URL")
val request = Request.Builder().url(websocketUrl).build()
StandardHermesSpeechStream(
@@ -195,6 +203,12 @@ class StandardHermesVoiceClient(
private fun dashboardBaseUrl(): String? =
dashboardUrlProvider()?.trim()?.trimEnd('/')?.takeIf { it.isNotBlank() }
private fun activeProfile(): String? =
profileProvider()?.trim()?.takeIf { it.isNotBlank() }
private fun dashboardAudioUrl(baseUrl: String, path: String) =
standardHermesAudioUrl(baseUrl, path, activeProfile())
private fun callClient(baseUrl: String): OkHttpClient =
standardHermesDashboardAudioClient(dashboardHttpClientProvider(baseUrl))
@@ -244,12 +258,6 @@ class StandardHermesVoiceClient(
return IOException(message)
}
private fun buildAudioDataUrl(audioFile: File): String {
val mimeType = mediaTypeForAudioFile(audioFile)
val encoded = Base64.getEncoder().encodeToString(audioFile.readBytes())
return "data:$mimeType;base64,$encoded"
}
private fun mediaTypeForAudioFile(file: File): String =
when (file.extension.lowercase()) {
"wav" -> "audio/wav"
@@ -294,6 +302,58 @@ class StandardHermesVoiceClient(
}
}
/**
* JSON request body for upstream `/api/audio/transcribe`.
*
* The endpoint requires a Base64 data URL inside JSON rather than multipart
* upload. Encoding into [BufferedSink] keeps peak memory bounded by the copy
* buffer instead of materializing several 33+ MB representations at once.
*/
internal fun standardHermesTranscriptionRequestBody(
audioFile: File,
mimeType: String,
): RequestBody {
val prefix = "{\"data_url\":\"data:$mimeType;base64,"
val suffix = "\",\"mime_type\":\"$mimeType\"}"
val contentType = "application/json".toMediaType()
val fileLength = audioFile.length()
val encodedLength = ((fileLength + 2L) / 3L) * 4L
val bodyLength = prefix.toByteArray(Charsets.UTF_8).size.toLong() +
encodedLength +
suffix.toByteArray(Charsets.UTF_8).size.toLong()
return object : RequestBody() {
override fun contentType() = contentType
override fun contentLength(): Long = bodyLength
override fun writeTo(sink: BufferedSink) {
sink.writeUtf8(prefix)
Base64.getEncoder().wrap(NonClosingSinkOutputStream(sink)).use { encoded ->
audioFile.inputStream().use { input ->
input.copyTo(encoded)
}
}
sink.writeUtf8(suffix)
}
}
}
/** Lets the Base64 encoder finish padding without closing OkHttp's request sink. */
private class NonClosingSinkOutputStream(
private val sink: BufferedSink,
) : OutputStream() {
override fun write(value: Int) {
sink.writeByte(value)
}
override fun write(bytes: ByteArray, offset: Int, length: Int) {
sink.write(bytes, offset, length)
}
override fun close() = Unit
}
private class StandardHermesSpeechStream(
private val request: Request,
private val callbacks: VoiceSpeechStreamCallbacks,
@@ -0,0 +1,181 @@
package com.hermesandroid.relay.petdex
import kotlinx.coroutines.CancellationException
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
class PetdexCatalogClient internal constructor(
private val fetcher: PetdexFetcher,
private val nowMs: () -> Long,
) {
constructor() : this(SecurePetdexFetcher(), System::currentTimeMillis)
@Volatile
private var cached: CachedCatalog? = null
suspend fun fetchCatalog(forceRefresh: Boolean = false): List<PetdexPet> {
val now = nowMs()
cached?.takeIf { !forceRefresh && now - it.loadedAtMs < CACHE_TTL_MS }?.let { return it.pets }
val pets = runCatching {
val bytes = fetcher.fetch(V2_URL, MAX_V2_BYTES, PetdexRemoteKind.Catalog)
PetdexCatalogParser.parseV2(bytes.decodeToString())
}.getOrElse { v2Error ->
if (v2Error is CancellationException) throw v2Error
runCatching {
val bytes = fetcher.fetch(V1_URL, MAX_V1_BYTES, PetdexRemoteKind.Catalog)
PetdexCatalogParser.parseV1(bytes.decodeToString())
}.getOrElse { v1Error ->
if (v1Error is CancellationException) throw v1Error
throw PetdexException("Couldn't load the Petdex catalog.", v1Error).also {
it.addSuppressed(v2Error)
}
}
}
cached = CachedCatalog(nowMs(), pets)
return pets
}
fun clearCache() {
cached = null
}
private data class CachedCatalog(val loadedAtMs: Long, val pets: List<PetdexPet>)
private companion object {
const val V2_URL = "https://petdex.dev/api/manifest/v2"
const val V1_URL = "https://petdex.dev/api/manifest"
const val CACHE_TTL_MS = 5 * 60 * 1000L
const val MAX_V2_BYTES = 4L * 1024 * 1024
const val MAX_V1_BYTES = 8L * 1024 * 1024
}
}
internal object PetdexCatalogParser {
private const val MAX_CATALOG_ITEMS = 10_000
private const val MAX_MANIFEST_FIELDS = 32
private const val MAX_FIELD_NAME_LENGTH = 64
private const val MAX_ROW_FIELDS = 32
private const val MAX_DISPLAY_NAME_LENGTH = 256
private const val MAX_KIND_LENGTH = 64
private const val MAX_CREATOR_LENGTH = 256
private const val MAX_ASSET_URL_LENGTH = 2_048
private val slugPattern = Regex("[a-z0-9][a-z0-9-]{0,127}")
private val json = Json { ignoreUnknownKeys = true }
fun parseV2(raw: String): List<PetdexPet> {
val manifest = json.decodeFromString(V2Manifest.serializer(), raw)
if (manifest.v != 2) throw PetdexException("Unsupported Petdex catalog version.")
if (manifest.pets.size > MAX_CATALOG_ITEMS) throw PetdexException("Petdex catalog has too many entries.")
if (manifest.fields.isEmpty() || manifest.fields.size > MAX_MANIFEST_FIELDS ||
manifest.fields.any { it.isBlank() || it.length > MAX_FIELD_NAME_LENGTH }
) {
throw PetdexException("Petdex catalog has invalid fields.")
}
val fieldIndex = manifest.fields.withIndex().associate { it.value to it.index }
if (!fieldIndex.keys.containsAll(REQUIRED_V2_FIELDS)) {
throw PetdexException("Petdex catalog is missing required fields.")
}
val base = manifest.assetBase.toHttpUrlOrNull()
?.takeIf { PetdexUrlPolicy.isTrusted(it.toString(), PetdexRemoteKind.Asset) }
?: throw PetdexException("Petdex catalog has an untrusted asset base.")
return manifest.pets.mapNotNull row@{ row ->
if (row.size > MAX_ROW_FIELDS || row.size > manifest.fields.size) return@row null
fun value(name: String): String = fieldIndex[name]
?.let(row::getOrNull)
?.jsonPrimitive
?.contentOrNull
.orEmpty()
buildPet(
slug = value("slug"),
displayName = value("displayName"),
kind = value("kind"),
submittedBy = value("submittedBy"),
spritesheetUrl = resolveAsset(base.toString(), value("spritesheet")),
petJsonUrl = resolveAsset(base.toString(), value("petJson")),
zipUrl = value("zip").takeIf(String::isNotBlank)?.let { resolveAsset(base.toString(), it) },
)
}
}
fun parseV1(raw: String): List<PetdexPet> {
val manifest = json.decodeFromString(V1Manifest.serializer(), raw)
if (manifest.pets.size > MAX_CATALOG_ITEMS) throw PetdexException("Petdex catalog has too many entries.")
return manifest.pets.mapNotNull { entry ->
buildPet(
slug = entry.slug,
displayName = entry.displayName,
kind = entry.kind,
submittedBy = entry.submittedBy.orEmpty(),
spritesheetUrl = entry.spritesheetUrl,
petJsonUrl = entry.petJsonUrl,
zipUrl = entry.zipUrl?.takeIf(String::isNotBlank),
)
}
}
private fun buildPet(
slug: String,
displayName: String,
kind: String,
submittedBy: String,
spritesheetUrl: String,
petJsonUrl: String,
zipUrl: String?,
): PetdexPet? {
if (!slugPattern.matches(slug)) return null
if (displayName.length > MAX_DISPLAY_NAME_LENGTH || kind.length > MAX_KIND_LENGTH ||
submittedBy.length > MAX_CREATOR_LENGTH || spritesheetUrl.length > MAX_ASSET_URL_LENGTH ||
petJsonUrl.length > MAX_ASSET_URL_LENGTH || (zipUrl?.length ?: 0) > MAX_ASSET_URL_LENGTH
) {
return null
}
if (!PetdexUrlPolicy.isTrusted(spritesheetUrl, PetdexRemoteKind.Asset)) return null
if (!PetdexUrlPolicy.isTrusted(petJsonUrl, PetdexRemoteKind.Asset)) return null
if (zipUrl != null && !PetdexUrlPolicy.isTrusted(zipUrl, PetdexRemoteKind.Asset)) return null
return PetdexPet(
slug = slug,
displayName = displayName.ifBlank { slug },
kind = kind.ifBlank { "pet" },
submittedBy = submittedBy,
spritesheetUrl = spritesheetUrl,
petJsonUrl = petJsonUrl,
zipUrl = zipUrl,
)
}
private fun resolveAsset(base: String, reference: String): String {
if (reference.isBlank()) return ""
val absolute = reference.toHttpUrlOrNull()
if (absolute != null) return absolute.toString()
return base.toHttpUrlOrNull()?.resolve(reference)?.toString().orEmpty()
}
private val REQUIRED_V2_FIELDS = setOf("slug", "displayName", "spritesheet", "petJson")
}
@Serializable
private data class V2Manifest(
val v: Int,
val assetBase: String,
val fields: List<String>,
val pets: List<List<JsonElement>>,
)
@Serializable
private data class V1Manifest(val pets: List<V1Pet> = emptyList())
@Serializable
private data class V1Pet(
val slug: String = "",
val displayName: String = "",
val kind: String = "",
val submittedBy: String? = null,
val spritesheetUrl: String = "",
val petJsonUrl: String = "",
val zipUrl: String? = null,
)
@@ -0,0 +1,106 @@
package com.hermesandroid.relay.petdex
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import java.io.ByteArrayOutputStream
import java.io.InputStream
import java.util.concurrent.TimeUnit
internal enum class PetdexRemoteKind { Catalog, Asset }
internal fun interface PetdexFetcher {
suspend fun fetch(url: String, maxBytes: Long, kind: PetdexRemoteKind): ByteArray
}
internal object PetdexUrlPolicy {
private val catalogHosts = setOf("petdex.dev", "assets.petdex.dev")
private val assetHosts = setOf("assets.petdex.dev")
fun isTrusted(url: String, kind: PetdexRemoteKind): Boolean {
val parsed = url.toHttpUrlOrNull() ?: return false
if (!parsed.isHttps) return false
if (parsed.port != 443 || parsed.username.isNotEmpty() || parsed.password.isNotEmpty()) return false
val hosts = if (kind == PetdexRemoteKind.Catalog) catalogHosts else assetHosts
return parsed.host in hosts
}
}
internal class SecurePetdexFetcher(
client: OkHttpClient = OkHttpClient.Builder()
.connectTimeout(10, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.followRedirects(false)
.followSslRedirects(false)
.build(),
) : PetdexFetcher {
private val http = client.newBuilder()
.followRedirects(false)
.followSslRedirects(false)
.build()
override suspend fun fetch(url: String, maxBytes: Long, kind: PetdexRemoteKind): ByteArray =
withContext(Dispatchers.IO) {
var current = trustedUrl(url, kind)
repeat(MAX_REDIRECTS + 1) { redirectCount ->
val request = Request.Builder()
.url(current)
.header("User-Agent", "Hermes-Relay-Android-Petdex")
.get()
.build()
http.newCall(request).execute().use { response ->
if (response.code in 300..399) {
if (redirectCount == MAX_REDIRECTS) throw PetdexException("Too many Petdex redirects.")
current = redirectTarget(response, current, kind)
} else {
if (!response.isSuccessful) throw PetdexException("Petdex request failed (${response.code}).")
return@withContext readBounded(
response.body.byteStream(),
response.body.contentLength(),
maxBytes,
)
}
}
}
throw PetdexException("Too many Petdex redirects.")
}
private fun trustedUrl(raw: String, kind: PetdexRemoteKind): HttpUrl {
if (!PetdexUrlPolicy.isTrusted(raw, kind)) throw PetdexException("Untrusted Petdex URL.")
return raw.toHttpUrlOrNull() ?: throw PetdexException("Invalid Petdex URL.")
}
private fun redirectTarget(response: Response, current: HttpUrl, kind: PetdexRemoteKind): HttpUrl {
val location = response.header("Location") ?: throw PetdexException("Petdex redirect had no destination.")
val resolved = current.resolve(location) ?: throw PetdexException("Invalid Petdex redirect.")
return trustedUrl(resolved.toString(), kind)
}
private companion object {
const val MAX_REDIRECTS = 3
}
}
internal fun readBounded(input: InputStream, declaredLength: Long, maxBytes: Long): ByteArray {
if (declaredLength < -1L) throw PetdexException("Petdex response had an invalid length.")
if (maxBytes <= 0L || declaredLength > maxBytes) throw PetdexException("Petdex download is too large.")
val initial = when {
declaredLength in 1..maxBytes -> declaredLength.toInt()
else -> minOf(maxBytes, 32L * 1024L).toInt()
}
val out = ByteArrayOutputStream(initial)
val buffer = ByteArray(8 * 1024)
var total = 0L
while (true) {
val read = input.read(buffer)
if (read < 0) break
total += read
if (total > maxBytes) throw PetdexException("Petdex download is too large.")
out.write(buffer, 0, read)
}
return out.toByteArray()
}
@@ -0,0 +1,256 @@
package com.hermesandroid.relay.petdex
import android.content.Context
import android.graphics.BitmapFactory
import com.hermesandroid.relay.ui.components.avatar.PetClipSpec
import com.hermesandroid.relay.ui.components.avatar.PetLoader
import com.hermesandroid.relay.ui.components.avatar.PetReactiveSpec
import com.hermesandroid.relay.ui.components.avatar.PetSpec
import com.hermesandroid.relay.ui.components.avatar.toAvatar
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.serialization.Serializable
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import java.io.File
import java.util.UUID
class PetdexInstaller internal constructor(
private val fetcher: PetdexFetcher,
) {
constructor() : this(SecurePetdexFetcher())
suspend fun install(context: Context, pet: PetdexPet): PetdexInstallResult =
installTo(PetLoader.userDir(context), pet)
internal suspend fun installTo(petsDir: File, pet: PetdexPet): PetdexInstallResult = installMutex.withLock {
withContext(Dispatchers.IO) {
runCatching {
validateCatalogPet(pet)
val metadataBytes = fetcher.fetch(pet.petJsonUrl, MAX_METADATA_BYTES, PetdexRemoteKind.Asset)
val remoteMetadata = parseMetadata(metadataBytes)
val spriteBytes = fetcher.fetch(pet.spritesheetUrl, MAX_SPRITESHEET_BYTES, PetdexRemoteKind.Asset)
val image = inspectSpritesheet(spriteBytes)
val layout = PetdexAtlasLayout.fromDimensions(image.width, image.height)
?: throw PetdexException("That Petdex pet uses an unsupported spritesheet layout.")
val extension = image.extension
val spec = layout.toPetSpec(pet, remoteMetadata, "spritesheet.$extension")
installAtomically(petsDir, pet.installedAvatarId, spec, spriteBytes, extension)
PetdexInstallResult.Success(spec.id, spec.label)
}.getOrElse { error ->
if (error is CancellationException) throw error
PetdexInstallResult.Failure(error.message ?: "Couldn't install that Petdex pet.")
}
}
}
private fun validateCatalogPet(pet: PetdexPet) {
if (!PETDEX_SLUG.matches(pet.slug)) throw PetdexException("Invalid Petdex pet id.")
if (!PetdexUrlPolicy.isTrusted(pet.petJsonUrl, PetdexRemoteKind.Asset) ||
!PetdexUrlPolicy.isTrusted(pet.spritesheetUrl, PetdexRemoteKind.Asset)
) {
throw PetdexException("Untrusted Petdex asset URL.")
}
}
private fun parseMetadata(bytes: ByteArray): PetdexMetadata {
val metadata = try {
json.decodeFromString(PetdexMetadata.serializer(), bytes.decodeToString())
} catch (t: Throwable) {
throw PetdexException("Petdex metadata isn't valid JSON.", t)
}
if (metadata.id.length > MAX_METADATA_ID_LENGTH ||
metadata.displayName.length > MAX_METADATA_NAME_LENGTH ||
metadata.description.length > MAX_METADATA_DESCRIPTION_LENGTH
) {
throw PetdexException("Petdex metadata fields are too large.")
}
return metadata
}
private fun inspectSpritesheet(bytes: ByteArray): InspectedImage {
val extension = when {
bytes.hasPrefix(PNG_MAGIC) -> "png"
bytes.hasWebpMagic() -> "webp"
else -> throw PetdexException("Petdex spritesheet isn't PNG or WebP.")
}
val options = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeByteArray(bytes, 0, bytes.size, options)
if (options.outWidth <= 0 || options.outHeight <= 0) {
throw PetdexException("Petdex spritesheet couldn't be decoded.")
}
val pixels = options.outWidth.toLong() * options.outHeight.toLong()
if (pixels > MAX_SPRITESHEET_PIXELS) throw PetdexException("Petdex spritesheet is too large.")
return InspectedImage(options.outWidth, options.outHeight, extension)
}
private fun installAtomically(
petsDir: File,
avatarId: String,
spec: PetSpec,
spriteBytes: ByteArray,
extension: String,
) {
petsDir.mkdirs()
val staging = File(petsDir, ".petdex-install-${UUID.randomUUID()}")
val backup = File(petsDir, ".petdex-backup-${UUID.randomUUID()}")
val target = File(petsDir, avatarId)
try {
check(staging.mkdir()) { "Couldn't prepare Petdex install." }
File(staging, "spritesheet.$extension").writeBytes(spriteBytes)
File(staging, "pet.json").writeText(json.encodeToString(spec))
spec.toAvatar(staging) // Validate the exact installed representation before swapping it in.
if (target.exists() && !target.renameTo(backup)) {
throw PetdexException("Couldn't replace the existing Petdex pet.")
}
if (!staging.renameTo(target)) {
if (backup.exists()) backup.renameTo(target)
throw PetdexException("Couldn't finish the Petdex install.")
}
backup.deleteRecursively()
} finally {
staging.deleteRecursively()
if (backup.exists() && !target.exists()) backup.renameTo(target)
if (backup.exists() && target.exists()) backup.deleteRecursively()
}
}
private data class InspectedImage(val width: Int, val height: Int, val extension: String)
private companion object {
val PETDEX_SLUG = Regex("[a-z0-9][a-z0-9-]{0,127}")
val PNG_MAGIC = byteArrayOf(0x89.toByte(), 0x50, 0x4E, 0x47)
const val MAX_METADATA_BYTES = 256L * 1024
const val MAX_METADATA_ID_LENGTH = 128
const val MAX_METADATA_NAME_LENGTH = 256
const val MAX_METADATA_DESCRIPTION_LENGTH = 4_096
const val MAX_SPRITESHEET_BYTES = 32L * 1024 * 1024
const val MAX_SPRITESHEET_PIXELS = 16L * 1024 * 1024
val installMutex = Mutex()
val json = Json {
ignoreUnknownKeys = true
prettyPrint = true
}
}
}
@Serializable
internal data class PetdexMetadata(
val id: String = "",
val displayName: String = "",
val description: String = "",
)
internal data class PetdexAtlasLayout(
val columns: Int,
val rows: Map<String, Int>,
val frameCounts: Map<String, Int> = emptyMap(),
) {
fun toPetSpec(pet: PetdexPet, metadata: PetdexMetadata, sheetName: String): PetSpec {
fun clip(rowName: String): PetClipSpec? = rows[rowName]?.let { row ->
val frameCount = frameCounts[rowName] ?: LEGACY_FRAMES_PER_STATE
val loopDurationMs = PETDEX_CURRENT_LOOP_DURATIONS_MS[rowName]
?: LEGACY_LOOP_DURATION_MS
PetClipSpec(
sheet = sheetName,
frameWidth = FRAME_WIDTH,
frameHeight = FRAME_HEIGHT,
frameCount = minOf(frameCount, columns),
startFrame = row * columns,
fps = frameCount * 1000f / loopDurationMs,
)
}
requireNotNull(clip("idle"))
// Preserve the source atlas taxonomy. PetLoader owns the backwards-
// compatible semantic mapping from these upstream names to Android
// activity and locomotion, so directional travel rows are not discarded
// or confused with the in-place `running` agent-work row.
val states = rows.keys.mapNotNull { rowName ->
clip(rowName)?.let { rowName to it }
}.toMap()
return PetSpec(
schemaVersion = 1,
id = pet.installedAvatarId,
label = pet.displayName.ifBlank { metadata.displayName.ifBlank { pet.slug } },
description = metadata.description,
source = "petdex",
sourceUrl = pet.sourceUrl,
creator = pet.submittedBy,
reactive = PetReactiveSpec(voice = true, tools = true, intensity = false),
states = states,
)
}
companion object {
const val FRAME_WIDTH = 192
const val FRAME_HEIGHT = 208
private const val LEGACY_FRAMES_PER_STATE = 6
private const val LEGACY_LOOP_DURATION_MS = 1100f
private val CURRENT_ROWS = mapOf(
"idle" to 0,
"running-right" to 1,
"running-left" to 2,
"waving" to 3,
"jumping" to 4,
"failed" to 5,
"waiting" to 6,
"running" to 7,
"review" to 8,
)
private val LEGACY_ROWS = mapOf(
"idle" to 0,
"wave" to 1,
"run" to 2,
"failed" to 3,
"review" to 4,
"jump" to 5,
)
fun fromDimensions(width: Int, height: Int): PetdexAtlasLayout? = when {
width == 8 * FRAME_WIDTH &&
(height == 9 * FRAME_HEIGHT || height == 11 * FRAME_HEIGHT) ->
PetdexAtlasLayout(8, CURRENT_ROWS, PETDEX_CURRENT_FRAME_COUNTS)
width == 9 * FRAME_WIDTH && height == 8 * FRAME_HEIGHT -> PetdexAtlasLayout(9, LEGACY_ROWS)
else -> null
}
}
}
/** Canonical used cells in the current Codex/Petdex 8-column atlas. */
internal val PETDEX_CURRENT_FRAME_COUNTS: Map<String, Int> = mapOf(
"idle" to 6,
"running-right" to 8,
"running-left" to 8,
"waving" to 4,
"jumping" to 5,
"failed" to 8,
"waiting" to 6,
"running" to 6,
"review" to 6,
)
/** Total authored duration of each current row; Android approximates variable frame timing with average fps. */
internal val PETDEX_CURRENT_LOOP_DURATIONS_MS: Map<String, Float> = mapOf(
"idle" to 1100f,
"running-right" to 1060f,
"running-left" to 1060f,
"waving" to 700f,
"jumping" to 840f,
"failed" to 1220f,
"waiting" to 1010f,
"running" to 820f,
"review" to 1030f,
)
private fun ByteArray.hasPrefix(prefix: ByteArray): Boolean =
size >= prefix.size && prefix.indices.all { this[it] == prefix[it] }
private fun ByteArray.hasWebpMagic(): Boolean =
size >= 12 &&
copyOfRange(0, 4).contentEquals("RIFF".encodeToByteArray()) &&
copyOfRange(8, 12).contentEquals("WEBP".encodeToByteArray())
@@ -0,0 +1,22 @@
package com.hermesandroid.relay.petdex
/** Public Petdex catalog entry. Installing is always an explicit user action. */
data class PetdexPet(
val slug: String,
val displayName: String,
val kind: String,
val submittedBy: String,
val spritesheetUrl: String,
val petJsonUrl: String,
val zipUrl: String? = null,
) {
val sourceUrl: String get() = "https://petdex.dev/pets/$slug"
val installedAvatarId: String get() = "petdex-$slug"
}
sealed interface PetdexInstallResult {
data class Success(val avatarId: String, val label: String) : PetdexInstallResult
data class Failure(val reason: String) : PetdexInstallResult
}
internal class PetdexException(message: String, cause: Throwable? = null) : Exception(message, cause)
@@ -0,0 +1,264 @@
package com.hermesandroid.relay.plugins.document
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
/**
* A declarative Android surface supplied by a Hermes plugin.
*
* Documents contain no executable code, URLs, Android intents, or gateway calls. The host
* renders the bounded element vocabulary and decides how (or whether) to handle actions and
* asset references.
*/
@Serializable
data class PluginDocument(
val schemaVersion: Int = CURRENT_SCHEMA_VERSION,
val pages: List<PluginPage>,
val initialState: Map<String, PluginValue> = emptyMap(),
/** Host-owned revision for generated pages; ordinary installed plugins may omit it. */
@SerialName("host_revision") val hostRevision: Int? = null,
) {
companion object {
const val CURRENT_SCHEMA_VERSION = 1
}
}
@Serializable
data class PluginPage(
val id: String,
val title: PluginText,
val content: PluginElement,
)
@Serializable
sealed interface PluginValue {
@Serializable
@SerialName("string")
data class StringValue(val value: String) : PluginValue
@Serializable
@SerialName("boolean")
data class BooleanValue(val value: Boolean) : PluginValue
@Serializable
@SerialName("number")
data class NumberValue(val value: Double) : PluginValue
@Serializable
@SerialName("null")
data object NullValue : PluginValue
}
@Serializable
sealed interface PluginText {
@Serializable
@SerialName("literal")
data class Literal(val value: String) : PluginText
@Serializable
@SerialName("binding")
data class Binding(
val key: String,
val fallback: String = "",
) : PluginText
}
@Serializable
sealed interface PluginCondition {
@Serializable
@SerialName("truthy")
data class Truthy(val key: String) : PluginCondition
@Serializable
@SerialName("equals")
data class Equals(val key: String, val value: PluginValue) : PluginCondition
@Serializable
@SerialName("not")
data class Not(val condition: PluginCondition) : PluginCondition
@Serializable
@SerialName("all")
data class All(val conditions: List<PluginCondition>) : PluginCondition
@Serializable
@SerialName("any")
data class Any(val conditions: List<PluginCondition>) : PluginCondition
}
@Serializable
data class PluginAnimation(
val enter: PluginAnimationKind = PluginAnimationKind.NONE,
val change: PluginAnimationKind = PluginAnimationKind.NONE,
val speed: PluginAnimationSpeed = PluginAnimationSpeed.NORMAL,
)
@Serializable
enum class PluginAnimationKind {
NONE,
FADE,
SCALE,
SLIDE_VERTICAL,
HIGHLIGHT,
}
@Serializable
enum class PluginAnimationSpeed { FAST, NORMAL, SLOW }
@Serializable
data class PluginAction(
val id: String,
val arguments: Map<String, PluginValue> = emptyMap(),
val confirmation: PluginText? = null,
val request: PluginActionRequest? = null,
)
@Serializable
data class PluginActionRequest(
val method: String = "POST",
val path: String,
)
@Serializable
sealed interface PluginElement {
val id: String
val visibleWhen: PluginCondition?
val animation: PluginAnimation
@Serializable
@SerialName("group")
data class Group(
override val id: String,
val direction: PluginDirection = PluginDirection.COLUMN,
val children: List<PluginElement>,
val spacing: PluginSpacing = PluginSpacing.MEDIUM,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("card")
data class Card(
override val id: String,
val child: PluginElement,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("text")
data class Text(
override val id: String,
val text: PluginText,
val style: PluginTextStyle = PluginTextStyle.BODY,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("badge")
data class Badge(
override val id: String,
val text: PluginText,
val tone: PluginTone = PluginTone.NEUTRAL,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("button")
data class Button(
override val id: String,
val label: PluginText,
val action: PluginAction,
val style: PluginButtonStyle = PluginButtonStyle.PRIMARY,
val enabledWhen: PluginCondition? = null,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("text_input")
data class TextInput(
override val id: String,
val label: PluginText,
val binding: String,
val placeholder: PluginText? = null,
val maxLength: Int = DEFAULT_INPUT_LIMIT,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("toggle")
data class Toggle(
override val id: String,
val label: PluginText,
val binding: String,
val enabledWhen: PluginCondition? = null,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("progress")
data class Progress(
override val id: String,
val valueBinding: String? = null,
val value: Double? = null,
val label: PluginText? = null,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("image")
data class Image(
override val id: String,
val asset: PluginAssetReference,
val contentDescription: PluginText,
val aspectRatio: Float = 16f / 9f,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("divider")
data class Divider(
override val id: String,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
@Serializable
@SerialName("spacer")
data class Spacer(
override val id: String,
val size: PluginSpacing = PluginSpacing.MEDIUM,
override val visibleWhen: PluginCondition? = null,
override val animation: PluginAnimation = PluginAnimation(),
) : PluginElement
companion object {
const val DEFAULT_INPUT_LIMIT = 2_000
}
}
/** An opaque, plugin-scoped asset id. The host resolves it; documents cannot supply URLs. */
@Serializable
data class PluginAssetReference(val id: String)
@Serializable
enum class PluginDirection { ROW, COLUMN }
@Serializable
enum class PluginSpacing { NONE, SMALL, MEDIUM, LARGE }
@Serializable
enum class PluginTextStyle { BODY, LABEL, TITLE, HEADLINE }
@Serializable
enum class PluginTone { NEUTRAL, INFO, SUCCESS, WARNING, DANGER }
@Serializable
enum class PluginButtonStyle { PRIMARY, SECONDARY, TEXT }
@@ -0,0 +1,183 @@
package com.hermesandroid.relay.plugins.document
data class PluginDocumentState(
val values: Map<String, PluginValue> = emptyMap(),
) {
operator fun get(key: String): PluginValue = values[key] ?: PluginValue.NullValue
fun updated(key: String, value: PluginValue): PluginDocumentState =
copy(values = values + (key to value))
fun resolve(text: PluginText): String = when (text) {
is PluginText.Literal -> text.value
is PluginText.Binding -> this[text.key].displayString() ?: text.fallback
}
fun matches(condition: PluginCondition?): Boolean = when (condition) {
null -> true
is PluginCondition.Truthy -> this[condition.key].isTruthy()
is PluginCondition.Equals -> this[condition.key] == condition.value
is PluginCondition.Not -> !matches(condition.condition)
is PluginCondition.All -> condition.conditions.all(::matches)
is PluginCondition.Any -> condition.conditions.any(::matches)
}
companion object {
fun from(document: PluginDocument): PluginDocumentState =
PluginDocumentState(document.initialState)
}
}
fun PluginValue.displayString(): String? = when (this) {
is PluginValue.StringValue -> value
is PluginValue.BooleanValue -> value.toString()
is PluginValue.NumberValue -> value.toString().removeSuffix(".0")
PluginValue.NullValue -> null
}
fun PluginValue.isTruthy(): Boolean = when (this) {
is PluginValue.StringValue -> value.isNotBlank()
is PluginValue.BooleanValue -> value
is PluginValue.NumberValue -> value != 0.0 && !value.isNaN()
PluginValue.NullValue -> false
}
sealed interface PluginDocumentValidation {
data object Valid : PluginDocumentValidation
data class Invalid(val errors: List<String>) : PluginDocumentValidation
}
/** Rejects pathological or ambiguous documents before they reach Compose. */
object PluginDocumentValidator {
const val MAX_PAGES = 32
const val MAX_ELEMENTS = 500
const val MAX_DEPTH = 16
const val MAX_CHILDREN = 100
const val MAX_TEXT_LENGTH = 16_000
const val MAX_STATE_ENTRIES = 250
private val safeIdentifier = Regex("^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
fun validate(document: PluginDocument): PluginDocumentValidation {
val errors = mutableListOf<String>()
if (document.schemaVersion != PluginDocument.CURRENT_SCHEMA_VERSION) {
errors += "Unsupported schema version: ${document.schemaVersion}"
}
if (document.pages.isEmpty() || document.pages.size > MAX_PAGES) {
errors += "Document must contain 1..$MAX_PAGES pages"
}
if (document.initialState.size > MAX_STATE_ENTRIES) {
errors += "Initial state exceeds $MAX_STATE_ENTRIES entries"
}
val ids = mutableSetOf<String>()
var elements = 0
document.initialState.keys.forEach { validateIdentifier("state key", it, errors) }
document.pages.forEach { page ->
validateIdentifier("page id", page.id, errors)
validateText(page.title, errors)
walk(page.content, 1) { element, depth ->
elements += 1
if (depth > MAX_DEPTH) errors += "Element ${element.id} exceeds depth $MAX_DEPTH"
validateIdentifier("element id", element.id, errors)
if (!ids.add(element.id)) errors += "Duplicate element id: ${element.id}"
validateElement(element, errors)
}
}
if (elements > MAX_ELEMENTS) errors += "Document exceeds $MAX_ELEMENTS elements"
return if (errors.isEmpty()) PluginDocumentValidation.Valid
else PluginDocumentValidation.Invalid(errors.distinct())
}
private fun validateElement(element: PluginElement, errors: MutableList<String>) {
validateCondition(element.visibleWhen, errors)
when (element) {
is PluginElement.Group -> if (element.children.size > MAX_CHILDREN) {
errors += "Element ${element.id} exceeds $MAX_CHILDREN children"
}
is PluginElement.Card -> Unit
is PluginElement.Text -> validateText(element.text, errors)
is PluginElement.Badge -> validateText(element.text, errors)
is PluginElement.Button -> {
validateText(element.label, errors)
validateIdentifier("action id", element.action.id, errors)
element.action.arguments.keys.forEach { validateIdentifier("action argument", it, errors) }
element.action.confirmation?.let { validateText(it, errors) }
validateCondition(element.enabledWhen, errors)
}
is PluginElement.TextInput -> {
validateText(element.label, errors)
element.placeholder?.let { validateText(it, errors) }
validateIdentifier("binding", element.binding, errors)
if (element.maxLength !in 1..PluginElement.DEFAULT_INPUT_LIMIT) {
errors += "Input ${element.id} has an invalid maxLength"
}
}
is PluginElement.Toggle -> {
validateText(element.label, errors)
validateIdentifier("binding", element.binding, errors)
validateCondition(element.enabledWhen, errors)
}
is PluginElement.Progress -> {
element.valueBinding?.let { validateIdentifier("binding", it, errors) }
element.label?.let { validateText(it, errors) }
if (element.value == null && element.valueBinding == null) {
errors += "Progress ${element.id} needs a value or binding"
}
if (element.value != null && (!element.value.isFinite() || element.value !in 0.0..1.0)) {
errors += "Progress ${element.id} value must be between 0 and 1"
}
}
is PluginElement.Image -> {
validateIdentifier("asset id", element.asset.id, errors)
validateText(element.contentDescription, errors)
if (!element.aspectRatio.isFinite() || element.aspectRatio !in 0.25f..4f) {
errors += "Image ${element.id} has an invalid aspect ratio"
}
}
is PluginElement.Divider, is PluginElement.Spacer -> Unit
}
}
private fun validateCondition(condition: PluginCondition?, errors: MutableList<String>) {
when (condition) {
null -> Unit
is PluginCondition.Truthy -> validateIdentifier("condition key", condition.key, errors)
is PluginCondition.Equals -> validateIdentifier("condition key", condition.key, errors)
is PluginCondition.Not -> validateCondition(condition.condition, errors)
is PluginCondition.All -> condition.conditions.forEach { validateCondition(it, errors) }
is PluginCondition.Any -> condition.conditions.forEach { validateCondition(it, errors) }
}
}
private fun validateText(text: PluginText, errors: MutableList<String>) {
when (text) {
is PluginText.Literal -> if (text.value.length > MAX_TEXT_LENGTH) {
errors += "Text exceeds $MAX_TEXT_LENGTH characters"
}
is PluginText.Binding -> {
validateIdentifier("text binding", text.key, errors)
if (text.fallback.length > MAX_TEXT_LENGTH) {
errors += "Text fallback exceeds $MAX_TEXT_LENGTH characters"
}
}
}
}
private fun validateIdentifier(label: String, value: String, errors: MutableList<String>) {
if (!safeIdentifier.matches(value)) errors += "Invalid $label: $value"
}
private fun walk(
element: PluginElement,
depth: Int,
visit: (PluginElement, Int) -> Unit,
) {
visit(element, depth)
when (element) {
is PluginElement.Group -> element.children.forEach { walk(it, depth + 1, visit) }
is PluginElement.Card -> walk(element.child, depth + 1, visit)
else -> Unit
}
}
}
@@ -0,0 +1,61 @@
package com.hermesandroid.relay.plugins.runtime
import com.hermesandroid.relay.network.upstream.DashboardApiClient
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.booleanOrNull
import kotlinx.serialization.json.contentOrNull
/**
* Discovers opt-in Android plugin surfaces through the vanilla upstream dashboard.
*
* The dashboard catalog is authoritative for plugin identity and active state. Only
* entries advertising a backend API are probed; a missing or malformed mobile manifest
* simply means that dashboard plugin has no Android surface.
*/
class PluginDiscoveryClient(
private val dashboard: DashboardApiClient,
) {
suspend fun discover(): Result<List<DiscoveredAndroidPlugin>> = runCatching {
val catalog = dashboard.getJsonElement(CATALOG_PATH).getOrThrow() as? JsonArray
?: error("Dashboard plugin catalog must be a JSON array")
catalog.mapNotNull(::parseCatalogEntry)
.mapNotNull { entry ->
val manifest = dashboard
.getJsonObject("/api/plugins/${entry.id}/mobile/manifest")
.getOrNull()
?: return@mapNotNull null
// The authenticated catalog supplies provenance. A backend document cannot
// claim another plugin's identity and inherit this plugin's namespace/grants.
if (manifest.string("id") != entry.id) return@mapNotNull null
DiscoveredAndroidPlugin(catalog = entry, manifest = manifest)
}
}
private fun parseCatalogEntry(element: kotlinx.serialization.json.JsonElement): AndroidPluginCatalogEntry? {
val root = element as? JsonObject ?: return null
val id = root.string("name") ?: return null
val exposesApi = root.boolean("has_api") == true || root.string("api") != null
if (!PluginIdentifiers.isValid(id) || !exposesApi) return null
return AndroidPluginCatalogEntry(
id = id,
label = root.string("label") ?: id,
description = root.string("description").orEmpty(),
version = root.string("version").orEmpty(),
icon = root.string("icon") ?: "Puzzle",
source = root.string("source").orEmpty(),
)
}
private fun JsonObject.string(key: String): String? =
(this[key] as? JsonPrimitive)?.contentOrNull?.trim()?.takeIf(String::isNotEmpty)
private fun JsonObject.boolean(key: String): Boolean? =
(this[key] as? JsonPrimitive)?.booleanOrNull
private companion object {
const val CATALOG_PATH = "/api/dashboard/plugins"
}
}
@@ -0,0 +1,93 @@
package com.hermesandroid.relay.plugins.runtime
/** Host-stamped context for plugin UI. It is never accepted from plugin JSON. */
data class PluginHostContext(
val connectionId: String,
val profileName: String?,
val sessionId: String?,
) {
init {
require(connectionId.isNotBlank()) { "connectionId must not be blank" }
require(sessionId == null || sessionId.isNotBlank()) { "sessionId must be null or non-blank" }
}
}
enum class PluginLifecycleChange {
CONNECTED,
DISCONNECTED,
CONNECTION_CHANGED,
PROFILE_CHANGED,
SESSION_CHANGED,
UNCHANGED,
}
data class PluginLifecycleSnapshot(
val context: PluginHostContext?,
val generation: Long,
val changedAtEpochMillis: Long,
val lastChange: PluginLifecycleChange,
)
/**
* Tracks provenance changes independently from UI navigation.
*
* A connection or profile change invalidates catalog/page ownership. A session change is
* metadata-only: it updates the context available to the active surface without replacing the
* authenticated Dashboard client or discarding the catalog.
*/
class PluginLifecycleTracker(
private val clock: () -> Long = System::currentTimeMillis,
) {
var snapshot: PluginLifecycleSnapshot = PluginLifecycleSnapshot(
context = null,
generation = 0,
changedAtEpochMillis = clock(),
lastChange = PluginLifecycleChange.UNCHANGED,
)
private set
fun update(next: PluginHostContext?): PluginLifecycleSnapshot {
val previous = snapshot.context
val change = when {
previous == next -> PluginLifecycleChange.UNCHANGED
previous == null && next != null -> PluginLifecycleChange.CONNECTED
previous != null && next == null -> PluginLifecycleChange.DISCONNECTED
previous?.connectionId != next?.connectionId -> PluginLifecycleChange.CONNECTION_CHANGED
previous?.profileName != next?.profileName -> PluginLifecycleChange.PROFILE_CHANGED
else -> PluginLifecycleChange.SESSION_CHANGED
}
if (change != PluginLifecycleChange.UNCHANGED) {
snapshot = PluginLifecycleSnapshot(
context = next,
generation = snapshot.generation + 1,
changedAtEpochMillis = clock(),
lastChange = change,
)
}
return snapshot
}
}
data class PluginCatalogPreview(
val context: PluginHostContext,
val pluginCount: Int,
val enabledPluginCount: Int,
val pageCount: Int,
val refreshedAtEpochMillis: Long,
val liveRefreshEnabled: Boolean,
) {
init {
require(pluginCount >= 0 && enabledPluginCount in 0..pluginCount)
require(pageCount >= 0)
}
}
object PluginCatalogRefreshPolicy {
const val VISIBLE_REFRESH_INTERVAL_MILLIS: Long = 5_000L
const val MIN_PAGE_REFRESH_SECONDS: Int = 5
const val MAX_PAGE_REFRESH_SECONDS: Int = 300
fun pageRefreshIntervalMillis(requestedSeconds: Int?): Long? = requestedSeconds
?.coerceIn(MIN_PAGE_REFRESH_SECONDS, MAX_PAGE_REFRESH_SECONDS)
?.times(1_000L)
}
@@ -0,0 +1,54 @@
package com.hermesandroid.relay.plugins.runtime
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
@Serializable
data class AndroidPluginManifest(
@SerialName("schema_version") val schemaVersion: Int = 1,
val id: String,
@SerialName("display_name") val displayName: String? = null,
val description: String? = null,
val version: String? = null,
@SerialName("min_host_api") val minHostApi: Int = 1,
@SerialName("default_enabled") val defaultEnabled: Boolean = false,
val contributions: List<AndroidPluginContribution> = emptyList(),
@SerialName("requested_capabilities")
val requestedCapabilities: List<AndroidPluginCapabilityRequest> = emptyList(),
val updates: AndroidPluginUpdateSource? = null,
)
@Serializable
data class AndroidPluginContribution(
val id: String,
val surface: String = "page",
val title: String,
val document: AndroidPluginDocumentEndpoint,
/** Optional host-rendered metadata used by Relay-generated declarative previews. */
val status: String? = null,
val lifecycle: String? = null,
val description: String? = null,
val revision: Int? = null,
val digest: String? = null,
)
@Serializable
data class AndroidPluginDocumentEndpoint(
val method: String = "GET",
val path: String,
)
@Serializable
data class AndroidPluginCapabilityRequest(
val id: String,
val reason: String,
val required: Boolean = false,
)
@Serializable
data class AndroidPluginUpdateSource(
@SerialName("poll_seconds") val pollSeconds: Int? = null,
)
const val ANDROID_PLUGIN_HOST_API_VERSION: Int = 1
const val PLUGIN_API_WRITE_CAPABILITY: String = "plugin.api.write"
@@ -0,0 +1,51 @@
package com.hermesandroid.relay.plugins.runtime
import kotlinx.serialization.json.JsonObject
/** Upstream dashboard metadata for a plugin whose backend can expose a mobile surface. */
data class AndroidPluginCatalogEntry(
val id: String,
val label: String,
val description: String,
val version: String,
val icon: String,
val source: String,
)
/**
* A mobile manifest with identity anchored to the authenticated dashboard catalog.
*
* [manifest] stays as JSON at this transport boundary. The renderer owns the versioned
* declarative document contract; the discovery layer must not let a manifest assert a
* different plugin identity or widen its API namespace.
*/
data class DiscoveredAndroidPlugin(
val catalog: AndroidPluginCatalogEntry,
val manifest: JsonObject,
)
/** Local preference scope. A null profile is the server-default profile context. */
data class PluginScope(
val connectionId: String,
val profileName: String?,
val pluginId: String,
) {
init {
require(connectionId.isNotBlank()) { "connectionId must not be blank" }
require(PluginIdentifiers.isValid(pluginId)) { "Invalid plugin id: $pluginId" }
}
}
data class PluginPreferenceState(
val enabled: Boolean = false,
val grants: Set<String> = emptySet(),
/** Distinguishes an explicit opt-out from a manifest's default enablement. */
val configured: Boolean = false,
)
/** Conservative identifier grammar shared by discovery, API routing, and persistence. */
object PluginIdentifiers {
private val pattern = Regex("^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$")
fun isValid(value: String): Boolean = pattern.matches(value)
}
@@ -0,0 +1,63 @@
package com.hermesandroid.relay.plugins.runtime
import android.content.Context
import androidx.datastore.core.DataStore
import androidx.datastore.preferences.core.Preferences
import androidx.datastore.preferences.core.booleanPreferencesKey
import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.stringSetPreferencesKey
import androidx.datastore.preferences.preferencesDataStore
import com.hermesandroid.relay.data.AgentDisplay
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
import java.security.MessageDigest
/** Durable, local-only plugin enablement and permission grants. */
class PluginPreferenceStore(
private val dataStore: DataStore<Preferences>,
) {
constructor(context: Context) : this(context.pluginPreferencesDataStore)
fun state(scope: PluginScope): Flow<PluginPreferenceState> {
val suffix = scope.keySuffix()
val enabledKey = booleanPreferencesKey("enabled_$suffix")
val grantsKey = stringSetPreferencesKey("grants_$suffix")
return dataStore.data.map { preferences ->
PluginPreferenceState(
enabled = preferences[enabledKey] ?: false,
grants = preferences[grantsKey].orEmpty(),
configured = enabledKey in preferences,
)
}
}
suspend fun setEnabled(scope: PluginScope, enabled: Boolean) {
val key = booleanPreferencesKey("enabled_${scope.keySuffix()}")
dataStore.edit { preferences -> preferences[key] = enabled }
}
suspend fun setGrants(scope: PluginScope, grants: Set<String>) {
require(grants.none(String::isBlank)) { "Plugin grants must not be blank" }
val key = stringSetPreferencesKey("grants_${scope.keySuffix()}")
dataStore.edit { preferences -> preferences[key] = grants.toSortedSet() }
}
suspend fun clear(scope: PluginScope) {
val suffix = scope.keySuffix()
dataStore.edit { preferences ->
preferences.remove(booleanPreferencesKey("enabled_$suffix"))
preferences.remove(stringSetPreferencesKey("grants_$suffix"))
}
}
private fun PluginScope.keySuffix(): String {
val profileKey = AgentDisplay.profileSessionKey(profileName)
val material = "$connectionId\u0000$profileKey\u0000$pluginId"
return MessageDigest.getInstance("SHA-256")
.digest(material.toByteArray(Charsets.UTF_8))
.joinToString("") { byte -> "%02x".format(byte) }
}
}
internal val Context.pluginPreferencesDataStore: DataStore<Preferences>
by preferencesDataStore(name = "plugin_preferences")
@@ -0,0 +1,96 @@
package com.hermesandroid.relay.plugins.runtime
import com.hermesandroid.relay.network.upstream.DashboardApiClient
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import java.net.URLEncoder
/**
* Namespace-confined access to one authenticated plugin backend.
*
* Callers provide path segments, never a URL. This rejects traversal and encoded path
* tricks before composing with [DashboardApiClient], while query values are encoded.
*/
class ScopedPluginApiClient(
private val pluginId: String,
private val dashboard: DashboardApiClient,
) {
init {
require(PluginIdentifiers.isValid(pluginId)) { "Invalid plugin id: $pluginId" }
}
suspend fun get(
relativePath: String,
query: Map<String, String> = emptyMap(),
): Result<JsonElement> = validateAndBuild(relativePath, query).fold(
onSuccess = { dashboard.getJsonElement(it) },
onFailure = { Result.failure(it) },
)
suspend fun post(relativePath: String, payload: JsonObject): Result<JsonObject> =
objectPath(relativePath).fold(
onSuccess = { dashboard.postJsonObject(it, payload) },
onFailure = { Result.failure(it) },
)
suspend fun put(relativePath: String, payload: JsonObject): Result<JsonObject> =
objectPath(relativePath).fold(
onSuccess = { dashboard.putJsonObject(it, payload) },
onFailure = { Result.failure(it) },
)
suspend fun patch(relativePath: String, payload: JsonObject): Result<JsonObject> =
objectPath(relativePath).fold(
onSuccess = { dashboard.patchJsonObject(it, payload) },
onFailure = { Result.failure(it) },
)
suspend fun delete(relativePath: String): Result<JsonObject> = objectPath(relativePath).fold(
onSuccess = { dashboard.deleteJsonObject(it) },
onFailure = { Result.failure(it) },
)
internal fun objectPath(relativePath: String): Result<String> =
validateAndBuild(relativePath, emptyMap())
private fun validateAndBuild(
relativePath: String,
query: Map<String, String>,
): Result<String> = runCatching {
val trimmed = relativePath.trim()
require(trimmed.isNotEmpty()) { "Plugin API path must not be empty" }
require(!trimmed.startsWith('/') && !trimmed.endsWith('/')) {
"Plugin API path must be relative and have no empty segments"
}
require('?' !in trimmed && '#' !in trimmed && '\\' !in trimmed && '%' !in trimmed) {
"Plugin API path must contain only plain path segments"
}
val segments = trimmed.split('/')
require(segments.all(::isSafeSegment)) { "Invalid plugin API path: $relativePath" }
buildString {
append("/api/plugins/")
append(pluginId)
append('/')
append(segments.joinToString("/"))
if (query.isNotEmpty()) {
append('?')
append(
query.entries.sortedBy { it.key }.joinToString("&") { (key, value) ->
"${encode(key)}=${encode(value)}"
},
)
}
}
}
private fun isSafeSegment(segment: String): Boolean =
segment.isNotEmpty() && segment != "." && segment != ".." && SEGMENT.matches(segment)
private fun encode(value: String): String =
URLEncoder.encode(value, Charsets.UTF_8.name()).replace("+", "%20")
private companion object {
val SEGMENT = Regex("^[a-zA-Z0-9._~-]+$")
}
}
@@ -0,0 +1,424 @@
package com.hermesandroid.relay.plugins.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.EnterTransition
import androidx.compose.animation.ExitTransition
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.animation.scaleIn
import androidx.compose.animation.scaleOut
import androidx.compose.animation.slideInVertically
import androidx.compose.animation.slideOutVertically
import androidx.compose.animation.core.Animatable
import androidx.compose.animation.core.tween
import androidx.compose.foundation.background
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.aspectRatio
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.verticalScroll
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.outlined.Image
import androidx.compose.material3.Button
import androidx.compose.material3.Card
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.Icon
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Surface
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.graphicsLayer
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
import com.hermesandroid.relay.plugins.document.PluginAction
import com.hermesandroid.relay.plugins.document.PluginAnimation
import com.hermesandroid.relay.plugins.document.PluginAnimationKind
import com.hermesandroid.relay.plugins.document.PluginAnimationSpeed
import com.hermesandroid.relay.plugins.document.PluginAssetReference
import com.hermesandroid.relay.plugins.document.PluginButtonStyle
import com.hermesandroid.relay.plugins.document.PluginDirection
import com.hermesandroid.relay.plugins.document.PluginDocument
import com.hermesandroid.relay.plugins.document.PluginDocumentState
import com.hermesandroid.relay.plugins.document.PluginElement
import com.hermesandroid.relay.plugins.document.PluginPage
import com.hermesandroid.relay.plugins.document.PluginSpacing
import com.hermesandroid.relay.plugins.document.PluginTextStyle
import com.hermesandroid.relay.plugins.document.PluginTone
import com.hermesandroid.relay.plugins.document.PluginValue
import com.hermesandroid.relay.ui.components.rememberAccessibleMotionState
sealed interface PluginInteraction {
data class ActionInvoked(
val elementId: String,
val action: PluginAction,
) : PluginInteraction
data class ValueChanged(
val elementId: String,
val key: String,
val value: PluginValue,
) : PluginInteraction
}
typealias PluginAssetContent = @Composable (
reference: PluginAssetReference,
contentDescription: String,
modifier: Modifier,
) -> Unit
/**
* Renders one complete plugin page using host-owned Compose components.
*
* [state] is controlled by the caller. Every local edit is emitted as a [PluginInteraction],
* making server reconciliation and permission checks explicit rather than hidden in UI code.
*/
@Composable
fun PluginPageRenderer(
document: PluginDocument,
pageId: String,
state: PluginDocumentState,
onInteraction: (PluginInteraction) -> Unit,
modifier: Modifier = Modifier,
assetContent: PluginAssetContent = { _, description, assetModifier ->
DefaultPluginAsset(description, assetModifier)
},
) {
val page = document.pages.firstOrNull { it.id == pageId }
if (page == null) {
return
}
PluginPageRenderer(
page = page,
state = state,
onInteraction = onInteraction,
modifier = modifier,
assetContent = assetContent,
)
}
@Composable
fun PluginPageRenderer(
page: PluginPage,
state: PluginDocumentState,
onInteraction: (PluginInteraction) -> Unit,
modifier: Modifier = Modifier,
assetContent: PluginAssetContent = { _, description, assetModifier ->
DefaultPluginAsset(description, assetModifier)
},
) {
Column(
modifier = modifier
.fillMaxSize()
.verticalScroll(rememberScrollState())
.padding(16.dp),
verticalArrangement = Arrangement.spacedBy(16.dp),
) {
Text(
text = state.resolve(page.title),
style = MaterialTheme.typography.headlineSmall,
fontWeight = FontWeight.SemiBold,
)
RenderElement(
element = page.content,
state = state,
onInteraction = onInteraction,
assetContent = assetContent,
modifier = Modifier.fillMaxWidth(),
)
}
}
@Composable
private fun RenderElement(
element: PluginElement,
state: PluginDocumentState,
onInteraction: (PluginInteraction) -> Unit,
assetContent: PluginAssetContent,
modifier: Modifier = Modifier,
) {
val visible = state.matches(element.visibleWhen)
val duration = element.animation.speed.milliseconds
val accessibleMotion = rememberAccessibleMotionState()
val animation = if (accessibleMotion.osAnimations && !accessibleMotion.touchExploration) {
element.animation
} else {
PluginAnimation()
}
AnimatedVisibility(
visible = visible,
modifier = modifier,
enter = animation.enter.enterTransition(duration),
exit = animation.enter.exitTransition(duration),
) {
val animatedModifier = Modifier.pluginChangeAnimation(
animation = animation,
stateFingerprint = state.values.hashCode(),
)
when (element) {
is PluginElement.Group -> PluginGroup(
element = element,
state = state,
onInteraction = onInteraction,
assetContent = assetContent,
modifier = animatedModifier,
)
is PluginElement.Card -> Card(modifier = animatedModifier.fillMaxWidth()) {
RenderElement(
element = element.child,
state = state,
onInteraction = onInteraction,
assetContent = assetContent,
modifier = Modifier.fillMaxWidth().padding(16.dp),
)
}
is PluginElement.Text -> Text(
text = state.resolve(element.text),
style = when (element.style) {
PluginTextStyle.BODY -> MaterialTheme.typography.bodyLarge
PluginTextStyle.LABEL -> MaterialTheme.typography.labelLarge
PluginTextStyle.TITLE -> MaterialTheme.typography.titleMedium
PluginTextStyle.HEADLINE -> MaterialTheme.typography.headlineSmall
},
modifier = animatedModifier,
)
is PluginElement.Badge -> PluginBadge(element, state, animatedModifier)
is PluginElement.Button -> PluginButton(element, state, onInteraction, animatedModifier)
is PluginElement.TextInput -> OutlinedTextField(
value = (state[element.binding] as? PluginValue.StringValue)?.value.orEmpty(),
onValueChange = { value ->
onInteraction(
PluginInteraction.ValueChanged(
elementId = element.id,
key = element.binding,
value = PluginValue.StringValue(value.take(element.maxLength)),
),
)
},
label = { Text(state.resolve(element.label)) },
placeholder = element.placeholder?.let { placeholder ->
{ Text(state.resolve(placeholder)) }
},
singleLine = false,
modifier = animatedModifier.fillMaxWidth(),
)
is PluginElement.Toggle -> Row(
modifier = animatedModifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(12.dp),
) {
Text(state.resolve(element.label), modifier = Modifier.weight(1f))
Switch(
checked = (state[element.binding] as? PluginValue.BooleanValue)?.value == true,
enabled = state.matches(element.enabledWhen),
onCheckedChange = { checked ->
onInteraction(
PluginInteraction.ValueChanged(
elementId = element.id,
key = element.binding,
value = PluginValue.BooleanValue(checked),
),
)
},
)
}
is PluginElement.Progress -> PluginProgress(element, state, animatedModifier)
is PluginElement.Image -> assetContent(
element.asset,
state.resolve(element.contentDescription),
animatedModifier.fillMaxWidth().aspectRatio(element.aspectRatio.coerceIn(0.25f, 4f)),
)
is PluginElement.Divider -> HorizontalDivider(modifier = animatedModifier.fillMaxWidth())
is PluginElement.Spacer -> Spacer(modifier = animatedModifier.height(element.size.dp))
}
}
}
@Composable
private fun PluginGroup(
element: PluginElement.Group,
state: PluginDocumentState,
onInteraction: (PluginInteraction) -> Unit,
assetContent: PluginAssetContent,
modifier: Modifier,
) {
if (element.direction == PluginDirection.COLUMN) {
Column(modifier = modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(element.spacing.dp)) {
element.children.forEach { child ->
RenderElement(child, state, onInteraction, assetContent, Modifier.fillMaxWidth())
}
}
} else {
Row(
modifier = modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
horizontalArrangement = Arrangement.spacedBy(element.spacing.dp),
verticalAlignment = Alignment.CenterVertically,
) {
element.children.forEach { child -> RenderElement(child, state, onInteraction, assetContent) }
}
}
}
@Composable
private fun PluginBadge(
element: PluginElement.Badge,
state: PluginDocumentState,
modifier: Modifier,
) {
val color = when (element.tone) {
PluginTone.NEUTRAL -> MaterialTheme.colorScheme.surfaceVariant
PluginTone.INFO -> MaterialTheme.colorScheme.primaryContainer
PluginTone.SUCCESS -> MaterialTheme.colorScheme.tertiaryContainer
PluginTone.WARNING -> MaterialTheme.colorScheme.secondaryContainer
PluginTone.DANGER -> MaterialTheme.colorScheme.errorContainer
}
Surface(modifier = modifier, color = color, shape = RoundedCornerShape(999.dp)) {
Text(
text = state.resolve(element.text),
style = MaterialTheme.typography.labelMedium,
modifier = Modifier.padding(horizontal = 10.dp, vertical = 5.dp),
)
}
}
@Composable
private fun PluginButton(
element: PluginElement.Button,
state: PluginDocumentState,
onInteraction: (PluginInteraction) -> Unit,
modifier: Modifier,
) {
val onClick = { onInteraction(PluginInteraction.ActionInvoked(element.id, element.action)) }
val enabled = state.matches(element.enabledWhen)
when (element.style) {
PluginButtonStyle.PRIMARY -> Button(onClick, modifier, enabled) { Text(state.resolve(element.label)) }
PluginButtonStyle.SECONDARY -> OutlinedButton(onClick, modifier, enabled) { Text(state.resolve(element.label)) }
PluginButtonStyle.TEXT -> TextButton(onClick, modifier, enabled) { Text(state.resolve(element.label)) }
}
}
@Composable
private fun PluginProgress(
element: PluginElement.Progress,
state: PluginDocumentState,
modifier: Modifier,
) {
val bound = element.valueBinding
?.let(state::get)
?.let { it as? PluginValue.NumberValue }
?.value
val progress = (bound ?: element.value ?: 0.0).toFloat().coerceIn(0f, 1f)
Column(modifier = modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(6.dp)) {
element.label?.let { Text(state.resolve(it), style = MaterialTheme.typography.labelMedium) }
LinearProgressIndicator(progress = { progress }, modifier = Modifier.fillMaxWidth())
}
}
@Composable
private fun DefaultPluginAsset(contentDescription: String, modifier: Modifier) {
Box(
modifier = modifier
.clip(RoundedCornerShape(12.dp))
.background(MaterialTheme.colorScheme.surfaceVariant),
contentAlignment = Alignment.Center,
) {
Icon(
imageVector = Icons.Outlined.Image,
contentDescription = contentDescription,
tint = MaterialTheme.colorScheme.onSurfaceVariant,
modifier = Modifier.size(32.dp),
)
}
}
@Composable
private fun Modifier.pluginChangeAnimation(
animation: PluginAnimation,
stateFingerprint: Int,
): Modifier {
val progress = remember { Animatable(1f) }
var composed by remember { mutableStateOf(false) }
LaunchedEffect(stateFingerprint, animation.change) {
if (animation.change == PluginAnimationKind.NONE) {
progress.snapTo(1f)
} else if (composed) {
progress.snapTo(0f)
progress.animateTo(1f, tween(animation.speed.milliseconds))
} else {
composed = true
}
}
if (animation.change == PluginAnimationKind.NONE) return this
val p = progress.value
return graphicsLayer {
when (animation.change) {
PluginAnimationKind.NONE -> Unit
PluginAnimationKind.FADE -> alpha = 0.65f + (0.35f * p)
PluginAnimationKind.SCALE -> {
scaleX = 0.96f + (0.04f * p)
scaleY = scaleX
}
PluginAnimationKind.SLIDE_VERTICAL -> translationY = 8.dp.toPx() * (1f - p)
PluginAnimationKind.HIGHLIGHT -> {
alpha = 0.82f + (0.18f * p)
scaleX = 0.98f + (0.02f * p)
scaleY = scaleX
}
}
}
}
private fun PluginAnimationKind.enterTransition(duration: Int): EnterTransition = when (this) {
PluginAnimationKind.NONE -> EnterTransition.None
PluginAnimationKind.FADE -> fadeIn(tween(duration))
PluginAnimationKind.SCALE -> fadeIn(tween(duration)) + scaleIn(tween(duration), initialScale = 0.96f)
PluginAnimationKind.SLIDE_VERTICAL -> fadeIn(tween(duration)) +
slideInVertically(tween(duration)) { height -> height.coerceAtMost(48) / 3 }
PluginAnimationKind.HIGHLIGHT -> fadeIn(tween(duration)) + scaleIn(tween(duration), initialScale = 0.98f)
}
private fun PluginAnimationKind.exitTransition(duration: Int): ExitTransition = when (this) {
PluginAnimationKind.NONE -> ExitTransition.None
PluginAnimationKind.FADE -> fadeOut(tween(duration))
PluginAnimationKind.SCALE -> fadeOut(tween(duration)) + scaleOut(tween(duration), targetScale = 0.96f)
PluginAnimationKind.SLIDE_VERTICAL -> fadeOut(tween(duration)) +
slideOutVertically(tween(duration)) { height -> height.coerceAtMost(48) / 3 }
PluginAnimationKind.HIGHLIGHT -> fadeOut(tween(duration)) + scaleOut(tween(duration), targetScale = 0.98f)
}
private val PluginAnimationSpeed.milliseconds: Int
get() = when (this) {
PluginAnimationSpeed.FAST -> 120
PluginAnimationSpeed.NORMAL -> 220
PluginAnimationSpeed.SLOW -> 360
}
private val PluginSpacing.dp
get() = when (this) {
PluginSpacing.NONE -> 0.dp
PluginSpacing.SMALL -> 8.dp
PluginSpacing.MEDIUM -> 16.dp
PluginSpacing.LARGE -> 24.dp
}
@@ -0,0 +1,203 @@
package com.hermesandroid.relay.runtime
import androidx.lifecycle.ViewModelProvider
import androidx.lifecycle.ViewModelStore
import com.hermesandroid.relay.HermesRelayApp
import com.hermesandroid.relay.data.VoicePreferencesRepository
import com.hermesandroid.relay.data.VoiceSettings
import com.hermesandroid.relay.network.relay.RelayVoiceClient
import com.hermesandroid.relay.viewmodel.ChatViewModel
import com.hermesandroid.relay.viewmodel.ConnectionViewModel
import com.hermesandroid.relay.viewmodel.VoiceViewModel
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Job
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
/**
* Application-lifetime owner for the runtime state shared by the normal app UI
* and service-started assistant turns.
*
* The runtime deliberately lives in the main application process. The isolated
* `:assistant_session` process exchanges snapshots and lifecycle commands over
* the assistant protocol and must never construct a second set of voice
* ViewModels or microphone collaborators.
*/
class HermesProcessRuntime internal constructor(
private val application: HermesRelayApp,
) {
private val viewModelStore = ViewModelStore()
private val viewModelProvider = ViewModelProvider(
viewModelStore,
ViewModelProvider.AndroidViewModelFactory.getInstance(application),
)
private val initializationMutex = Mutex()
private val activationLock = Any()
private var activationGeneration = 0L
private var currentActivationId: String? = null
private var activationJob: Job? = null
private val runtimeJob = SupervisorJob()
private val binder by lazy(LazyThreadSafetyMode.SYNCHRONIZED) {
HermesRuntimeBinder(application, this)
}
private val _initializationState =
MutableStateFlow(HermesRuntimeInitializationState.Uninitialized)
/**
* Process-lifetime scope for runtime binders and state bridges. UI work
* should continue to use its lifecycle-aware scopes.
*/
val coroutineScope: CoroutineScope =
CoroutineScope(runtimeJob + Dispatchers.Main.immediate)
/**
* Existing app ViewModels, now owned by the process runtime instead of an
* Activity. Each property remains lazy so merely starting the Application
* does not initialize network, chat, or microphone state.
*/
val connectionViewModel: ConnectionViewModel by lazy(LazyThreadSafetyMode.SYNCHRONIZED) {
viewModelProvider[ConnectionViewModel::class.java]
}
val chatViewModel: ChatViewModel by lazy(LazyThreadSafetyMode.SYNCHRONIZED) {
viewModelProvider[ChatViewModel::class.java]
}
val voiceViewModel: VoiceViewModel by lazy(LazyThreadSafetyMode.SYNCHRONIZED) {
viewModelProvider[VoiceViewModel::class.java]
}
val initializationState: StateFlow<HermesRuntimeInitializationState> =
_initializationState.asStateFlow()
val voiceActivationReadiness: StateFlow<HermesVoiceActivationReadiness>
get() = binder.voiceActivationReadiness
val assistantSnapshot: StateFlow<com.hermesandroid.relay.assistant.AssistantSessionSnapshot>
get() = binder.assistantSnapshot
/**
* Process-owned collaborators still consumed by Compose screens. Callers
* must await [ensureInitialized] before reading [relayVoiceClient].
*/
val relayVoiceClient: RelayVoiceClient
get() = binder.requireRelayVoiceClient()
val voicePreferences: VoicePreferencesRepository
get() = binder.voicePreferencesRepository
val voiceSettings: StateFlow<VoiceSettings>
get() = binder.voiceSettings
/**
* Runs the non-UI runtime wiring exactly once. A failed attempt returns the
* runtime to [HermesRuntimeInitializationState.Uninitialized] so a later
* foreground or assistant activation can retry cleanly.
*/
suspend fun ensureInitialized() {
if (_initializationState.value == HermesRuntimeInitializationState.Ready) return
initializationMutex.withLock {
if (_initializationState.value == HermesRuntimeInitializationState.Ready) return
_initializationState.value = HermesRuntimeInitializationState.Initializing
try {
withContext(Dispatchers.Main.immediate) {
binder.bind()
}
_initializationState.value = HermesRuntimeInitializationState.Ready
} catch (failure: Throwable) {
_initializationState.value = HermesRuntimeInitializationState.Uninitialized
throw failure
}
}
}
fun requestVoiceActivation(
activationId: String,
startNewSession: Boolean = true,
timeoutMs: Long = DEFAULT_VOICE_ACTIVATION_TIMEOUT_MS,
onFailure: (Throwable) -> Unit = {},
) {
val nextJob = synchronized(activationLock) {
// The assistant session process can replay the same activation while
// being recreated. That replay must not re-arm the recorder.
if (currentActivationId == activationId) return
activationJob?.cancel()
activationGeneration += 1
val generation = activationGeneration
currentActivationId = activationId
coroutineScope.launch(start = CoroutineStart.LAZY) {
try {
ensureInitialized()
binder.activateVoice(
startNewSession = startNewSession,
timeoutMs = timeoutMs,
isCurrent = {
synchronized(activationLock) {
activationGeneration == generation &&
currentActivationId == activationId
}
},
)
} catch (cancelled: CancellationException) {
throw cancelled
} catch (failure: Throwable) {
onFailure(failure)
}
}.also { activationJob = it }
}
nextJob.start()
}
fun cancelVoice() {
synchronized(activationLock) {
activationGeneration += 1
currentActivationId = null
activationJob?.cancel()
activationJob = null
}
if (_initializationState.value != HermesRuntimeInitializationState.Uninitialized) {
binder.cancelVoice()
}
}
/**
* Production Android processes are torn down as a unit. This explicit
* cleanup seam exists for local/instrumentation hosts that construct more
* than one Application instance in a single VM.
*/
internal fun clear() {
synchronized(activationLock) {
activationGeneration += 1
currentActivationId = null
activationJob?.cancel()
activationJob = null
}
if (_initializationState.value != HermesRuntimeInitializationState.Uninitialized) {
binder.clear()
}
coroutineScope.cancel()
viewModelStore.clear()
_initializationState.value = HermesRuntimeInitializationState.Uninitialized
}
private companion object {
const val DEFAULT_VOICE_ACTIVATION_TIMEOUT_MS = 20_000L
}
}
enum class HermesRuntimeInitializationState {
Uninitialized,
Initializing,
Ready,
}
@@ -0,0 +1,522 @@
package com.hermesandroid.relay.runtime
import androidx.lifecycle.viewModelScope
import com.hermesandroid.relay.HermesRelayApp
import com.hermesandroid.relay.assistant.AssistantAppSessionState
import com.hermesandroid.relay.assistant.AssistantSessionPhase
import com.hermesandroid.relay.assistant.AssistantSessionProtocol
import com.hermesandroid.relay.assistant.AssistantSessionSnapshot
import com.hermesandroid.relay.assistant.AssistantVoiceCommandCoordinator
import com.hermesandroid.relay.audio.BargeInListener
import com.hermesandroid.relay.audio.RealtimePcmPlayer
import com.hermesandroid.relay.audio.VadEngine
import com.hermesandroid.relay.audio.VoicePlayer
import com.hermesandroid.relay.audio.VoiceRecorder
import com.hermesandroid.relay.audio.VoiceSfxPlayer
import com.hermesandroid.relay.auth.AuthState
import com.hermesandroid.relay.data.AgentDisplay
import com.hermesandroid.relay.data.BargeInPreferencesRepository
import com.hermesandroid.relay.data.BuildFlavor
import com.hermesandroid.relay.data.EnhancedVoiceOverrides
import com.hermesandroid.relay.data.VoiceAudioRoute
import com.hermesandroid.relay.data.VoiceEngineMode
import com.hermesandroid.relay.data.VoicePreferencesRepository
import com.hermesandroid.relay.data.VoiceSettings
import com.hermesandroid.relay.network.relay.RelayVoiceAudioClientAdapter
import com.hermesandroid.relay.network.relay.RelayVoiceClient
import com.hermesandroid.relay.network.shared.AutoVoiceAudioClient
import com.hermesandroid.relay.network.upstream.StandardHermesVoiceClient
import com.hermesandroid.relay.viewmodel.StandardVoiceAvailability
import com.hermesandroid.relay.viewmodel.VoiceState
import java.util.concurrent.TimeUnit
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.flow.collectLatest
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.mapNotNull
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withTimeout
import okhttp3.OkHttpClient
/**
* Process-lifetime composition root for the chat and voice state machines.
*
* This owns only runtime behavior. Navigation, Activity lifecycle revalidation,
* and visual presentation remain UI responsibilities.
*/
internal class HermesRuntimeBinder(
private val application: HermesRelayApp,
private val runtime: HermesProcessRuntime,
) {
private val jobs = mutableListOf<Job>()
private var bound = false
val voicePreferencesRepository = VoicePreferencesRepository(application)
private val _voiceSettings = MutableStateFlow(VoiceSettings())
val voiceSettings: StateFlow<VoiceSettings> = _voiceSettings.asStateFlow()
private lateinit var relayVoiceClient: RelayVoiceClient
private val _voiceActivationReadiness =
MutableStateFlow<HermesVoiceActivationReadiness>(HermesVoiceActivationReadiness.Initializing)
val voiceActivationReadiness: StateFlow<HermesVoiceActivationReadiness> =
_voiceActivationReadiness.asStateFlow()
private val voiceSettingsHydrated = MutableStateFlow(false)
private val profileContextReady = MutableStateFlow(false)
private val _assistantSnapshot = MutableStateFlow(AssistantSessionSnapshot())
val assistantSnapshot: StateFlow<AssistantSessionSnapshot> = _assistantSnapshot.asStateFlow()
fun bind() {
if (bound) return
val connection = runtime.connectionViewModel
val chat = runtime.chatViewModel
val voice = runtime.voiceViewModel
relayVoiceClient = RelayVoiceClient(
context = application,
okHttpClient = OkHttpClient.Builder()
.readTimeout(2, TimeUnit.MINUTES)
.connectTimeout(15, TimeUnit.SECONDS)
.build(),
relayUrlProvider = { connection.effectiveRelayUrl.value },
relayRouteChangesProvider = {
connection.activeEndpoint.mapNotNull { it?.relay?.url }
},
routeProbeRequester = connection::probeNow,
profileNameProvider = {
AgentDisplay.profileRequestName(connection.selectedProfile.value?.name)
},
sessionTokenProvider = {
(connection.authState.value as? AuthState.Paired)?.token
},
apiBearerTokenProvider = connection::getApiKey,
)
val standardVoiceClient = StandardHermesVoiceClient(
context = application,
dashboardHttpClientProvider = connection::dashboardHttpClientForActive,
dashboardUrlProvider = connection::activeDashboardUrl,
profileProvider = {
AgentDisplay.profileRequestName(connection.selectedProfile.value?.name)
},
)
val voiceAudioClient = AutoVoiceAudioClient(
standardClient = standardVoiceClient,
relayClient = RelayVoiceAudioClientAdapter(
relayVoiceClient,
enhancedOverridesProvider = {
EnhancedVoiceOverrides.fromSettings(voiceSettings.value)
},
),
routeProvider = {
VoiceAudioRoute.fromStorage(voiceSettings.value.audioRoute)
},
standardReadyProvider = { connection.standardVoiceReady.value },
relayReadyProvider = { connection.relayVoiceReady.value },
)
voice.initialize(
voiceClient = relayVoiceClient,
voiceAudioClient = voiceAudioClient,
chatViewModel = chat,
recorder = VoiceRecorder(application, voice.viewModelScope),
player = VoicePlayer(application),
realtimePcmPlayer = RealtimePcmPlayer(application),
sfxPlayer = VoiceSfxPlayer(application),
bridgeMultiplexer = connection.multiplexer,
localBridgeDispatcher = if (BuildFlavor.isSideload) {
connection.bridgeCommandHandler::handleLocalCommand
} else {
null
},
voicePreferences = voicePreferencesRepository,
voiceRelayPreflight = connection::verifyRelayForVoice,
voiceHandoffReporter = connection::recordVoiceHandoff,
bargeInPreferences = BargeInPreferencesRepository(application),
vadEngineFactory = { VadEngine(application) },
bargeInListenerFactory = { vad, audioSessionIdProvider ->
BargeInListener.create(application, vad, audioSessionIdProvider)
},
)
bindChatDependencies()
bindProcessCollectors()
bound = true
}
private fun bindChatDependencies() {
val connection = runtime.connectionViewModel
val chat = runtime.chatViewModel
val voice = runtime.voiceViewModel
chat.initialize(connection.chatApiClient.value, connection.chatHandler)
chat.initializeMedia(
context = application,
relayHttpClient = connection.relayHttpClient,
mediaSettingsRepo = connection.mediaSettingsRepo,
mediaCacheWriter = connection.mediaCacheWriter,
)
chat.setSelectedProfileProvider { connection.selectedProfile.value }
chat.setIsolatedProfileApiProvider { connection.selectedProfileUsesIsolatedApiRoute() }
chat.setSessionProfileNameProvider { connection.effectiveSessionProfileName.value }
chat.setEffectiveProfileProvider {
AgentDisplay.effectiveProfile(
selectedProfile = connection.selectedProfile.value,
profiles = connection.agentProfiles.value,
)
}
chat.setDisplayProfileProvider { connection.effectiveDisplayProfile.value }
chat.setDisplayAliasProvider { connection.profileDisplayAlias.value }
chat.setProfileSessionLister { connection.listProfileScopedSessions() }
chat.setProfileMessageLoader(connection::loadProfileScopedMessages)
chat.setDashboardConfigLoader { connection.loadActiveDashboardConfig() }
chat.profileSessionDeleter = connection::deleteProfileScopedSession
chat.profileSessionRenamer = connection::renameProfileScopedSession
chat.onSessionChanged = connection::saveLastSessionId
chat.setDemoModeWiring(
isDemo = { connection.isDemoMode.value },
handler = { connection.chatHandler },
)
voice.chatNoticeSink = connection.chatHandler::addSystemNotice
connection.registerStreamCancelCallback(chat::cancelStream)
if (BuildFlavor.isSideload) {
connection.registerVoiceStopCallback(voice::exitVoiceMode)
}
chat.observeConnectionSwitches(connection.connectionSwitchEvents)
}
private fun bindProcessCollectors() {
val connection = runtime.connectionViewModel
val chat = runtime.chatViewModel
val voice = runtime.voiceViewModel
var boundCatalogConnectionId: String? = null
var lastAcquiredDashboardUrl: String? = null
jobs += runtime.coroutineScope.launch {
connection.authState.collect { auth ->
if (auth is AuthState.Paired) connection.reconnectIfStale()
}
}
jobs += runtime.coroutineScope.launch {
combine(connection.chatApiClient, connection.activeConnectionId) { client, id ->
client to id
}.collect { (client, connectionId) ->
if (chat.boundHandler === connection.chatHandler) {
if (boundCatalogConnectionId != connectionId) {
chat.resetConnectionCatalogs()
chat.updateApiClient(null)
} else {
chat.updateApiClient(client)
}
} else {
chat.initialize(client, connection.chatHandler)
}
boundCatalogConnectionId = connectionId
}
}
jobs += runtime.coroutineScope.launch {
chat.isStreaming.collect(connection::setChatStreaming)
}
jobs += runtime.coroutineScope.launch {
combine(
connection.activeConnectionId,
connection.selectedProfile,
) { connectionId, profile ->
connectionId to AgentDisplay.profileRequestName(profile?.name)
}.distinctUntilChanged().collectLatest { (connectionId, profileName) ->
voiceSettingsHydrated.value = false
voice.setVoicePrefsConnection(connectionId)
voicePreferencesRepository.setActiveScope(connectionId, profileName)
voice.onProfileChanged(profileName)
// Collect the repository flow directly. Its first value is read
// from DataStore for the selected scope; unlike a seeded
// StateFlow, it cannot falsely mark default settings hydrated.
voicePreferencesRepository.settings.collect { settings ->
_voiceSettings.value = settings
voiceSettingsHydrated.value = true
}
}
}
jobs += runtime.coroutineScope.launch {
combine(
connection.streamingEndpoint,
connection.serverCapabilities,
connection.gatewayAvailability,
connection.effectiveDashboardUrl,
) { preference, _, gateway, dashboardUrl ->
Triple(preference, gateway, dashboardUrl)
}.collectLatest { (preference, _, dashboardUrl) ->
if (
lastAcquiredDashboardUrl != null &&
lastAcquiredDashboardUrl != dashboardUrl
) {
delay(GATEWAY_ROUTE_SETTLE_MS)
}
val resolved = connection.resolveStreamingEndpoint(preference)
chat.streamingEndpoint = resolved
chat.sseFallbackEndpoint = connection.resolveSseStreamingEndpoint()
chat.updateGatewayClient(
if (resolved == "gateway") connection.activeGatewayChatClient() else null,
)
lastAcquiredDashboardUrl = dashboardUrl
}
}
jobs += runtime.coroutineScope.launch {
val contextInputs = combine(
connection.chatReady,
connection.activeConnectionId,
connection.effectiveSessionProfileName,
connection.lastSessionId,
) { ready, connectionId, profileName, sessionId ->
ProfileContextInputs(ready, connectionId, profileName, sessionId)
}
combine(contextInputs, connection.profileSelectionSettled) { inputs, settled ->
inputs.copy(profileSelectionSettled = settled)
}.collectLatest { inputs ->
profileContextReady.value = false
if (!inputs.chatReady) return@collectLatest
if (!inputs.profileSelectionSettled) delay(PROFILE_SETTLE_BACKSTOP_MS)
else delay(PROFILE_CONTEXT_COALESCE_MS)
chat.switchProfileContext(
contextKey = AgentDisplay.profileContextKey(
connectionId = inputs.connectionId,
profileName = inputs.profileName,
),
sessionId = inputs.sessionId,
)
chat.refreshSessions()
profileContextReady.value = true
}
}
jobs += runtime.coroutineScope.launch {
connection.parseToolAnnotations.collect { enabled ->
connection.chatHandler.parseToolAnnotations = enabled
}
}
jobs += runtime.coroutineScope.launch {
connection.showSystemMessages.collect { enabled ->
connection.chatHandler.showSystemMarkers = enabled
}
}
jobs += runtime.coroutineScope.launch {
val routeReadiness = combine(
voiceSettings,
connection.chatReady,
connection.standardVoiceAvailability,
connection.relayVoiceReady,
connection.profileSelectionSettled,
) { settings, chatReady, standard, relayReady, profileSettled ->
resolveVoiceActivationReadiness(
settings,
chatReady,
standard,
relayReady,
profileSettled,
)
}
combine(
routeReadiness,
voiceSettingsHydrated,
profileContextReady,
) { readiness, settingsReady, contextReady ->
when {
!settingsReady ->
HermesVoiceActivationReadiness.Waiting("Loading voice settings")
!contextReady &&
(readiness as? HermesVoiceActivationReadiness.Ready)?.route !=
HermesVoiceActivationRoute.Realtime ->
HermesVoiceActivationReadiness.Waiting("Restoring the Hermes chat context")
else -> readiness
}
}.collect { readiness ->
_voiceActivationReadiness.value = readiness
}
}
jobs += runtime.coroutineScope.launch {
voice.uiState.collect { state ->
val snapshot = AssistantSessionProtocol.snapshotFromVoiceState(state)
_assistantSnapshot.value = snapshot
if (!AssistantAppSessionState.active.value) return@collect
if (state.voiceMode) AssistantAppSessionState.markVoiceStarted()
if (state.voiceMode || AssistantAppSessionState.hasVoiceStarted()) {
AssistantSessionProtocol.publish(application, snapshot)
}
}
}
jobs += runtime.coroutineScope.launch {
AssistantVoiceCommandCoordinator.cancelRequest.collect { request ->
request ?: return@collect
cancelVoice()
AssistantVoiceCommandCoordinator.consume(request)
}
}
}
suspend fun activateVoice(
startNewSession: Boolean,
timeoutMs: Long,
isCurrent: () -> Boolean,
) {
check(bound) { "Hermes runtime is not initialized" }
val connection = runtime.connectionViewModel
val chat = runtime.chatViewModel
val voice = runtime.voiceViewModel
connection.reconnectIfStale()
connection.revalidate()
val readiness = withTimeout(timeoutMs) {
voiceActivationReadiness.first {
it is HermesVoiceActivationReadiness.Ready ||
it is HermesVoiceActivationReadiness.Blocked
}
}
if (readiness is HermesVoiceActivationReadiness.Blocked) {
error(readiness.reason)
}
readiness as HermesVoiceActivationReadiness.Ready
currentCoroutineContext().ensureActive()
check(isCurrent()) { "Assistant activation was superseded" }
// Re-apply scope before entry. The readiness collector already observed
// the scope's resolved settings, so Realtime prewarm cannot use defaults.
voice.setVoicePrefsConnection(connection.activeConnectionId.value)
voice.onProfileChanged(
AgentDisplay.profileRequestName(connection.selectedProfile.value?.name)
)
if (startNewSession) {
currentCoroutineContext().ensureActive()
check(isCurrent()) { "Assistant activation was superseded" }
chat.createNewChat()
}
currentCoroutineContext().ensureActive()
check(isCurrent()) { "Assistant activation was superseded" }
voice.enterVoiceMode()
currentCoroutineContext().ensureActive()
check(isCurrent()) { "Assistant activation was superseded" }
voice.startListening()
check(voice.uiState.value.state == VoiceState.Listening) {
voice.uiState.value.error ?: "Voice recorder did not enter Listening"
}
_voiceActivationReadiness.value = readiness
}
fun cancelVoice() {
runtime.voiceViewModel.exitVoiceMode()
}
fun requireRelayVoiceClient(): RelayVoiceClient {
check(bound && ::relayVoiceClient.isInitialized) {
"Call HermesProcessRuntime.ensureInitialized() before using voice clients"
}
return relayVoiceClient
}
fun clear() {
jobs.forEach { it.cancel() }
jobs.clear()
bound = false
_voiceActivationReadiness.value = HermesVoiceActivationReadiness.Initializing
_assistantSnapshot.value = AssistantSessionSnapshot(phase = AssistantSessionPhase.Closed)
}
private data class ProfileContextInputs(
val chatReady: Boolean,
val connectionId: String?,
val profileName: String?,
val sessionId: String?,
val profileSelectionSettled: Boolean = false,
)
private companion object {
const val PROFILE_SETTLE_BACKSTOP_MS = 2_500L
const val PROFILE_CONTEXT_COALESCE_MS = 160L
const val GATEWAY_ROUTE_SETTLE_MS = 750L
}
}
sealed interface HermesVoiceActivationReadiness {
data object Initializing : HermesVoiceActivationReadiness
data class Waiting(val reason: String) : HermesVoiceActivationReadiness
data class Ready(val route: HermesVoiceActivationRoute) : HermesVoiceActivationReadiness
data class Blocked(val reason: String) : HermesVoiceActivationReadiness
}
enum class HermesVoiceActivationRoute {
Standard,
RelayAudio,
Realtime,
}
internal fun resolveVoiceActivationReadiness(
settings: VoiceSettings,
chatReady: Boolean,
standardAvailability: StandardVoiceAvailability,
relayReady: Boolean,
profileSettled: Boolean,
): HermesVoiceActivationReadiness {
if (!profileSettled) {
return HermesVoiceActivationReadiness.Waiting("Loading the selected Hermes profile")
}
return when (VoiceEngineMode.fromStorage(settings.engineMode)) {
VoiceEngineMode.RealtimeAgent -> {
if (relayReady) {
HermesVoiceActivationReadiness.Ready(HermesVoiceActivationRoute.Realtime)
} else {
HermesVoiceActivationReadiness.Waiting("Waiting for the Relay realtime route")
}
}
VoiceEngineMode.HermesVoiceOutput -> {
if (!chatReady) {
return HermesVoiceActivationReadiness.Waiting("Waiting for Hermes chat")
}
when (VoiceAudioRoute.fromStorage(settings.audioRoute)) {
VoiceAudioRoute.Relay -> if (relayReady) {
HermesVoiceActivationReadiness.Ready(
HermesVoiceActivationRoute.RelayAudio
)
} else {
HermesVoiceActivationReadiness.Waiting("Waiting for Relay voice")
}
VoiceAudioRoute.Standard -> standardVoiceReadiness(standardAvailability)
VoiceAudioRoute.Auto -> when {
relayReady -> HermesVoiceActivationReadiness.Ready(
HermesVoiceActivationRoute.RelayAudio
)
standardAvailability == StandardVoiceAvailability.Ready ->
HermesVoiceActivationReadiness.Ready(
HermesVoiceActivationRoute.Standard
)
else -> standardVoiceReadiness(standardAvailability)
}
}
}
}
}
private fun standardVoiceReadiness(
availability: StandardVoiceAvailability,
): HermesVoiceActivationReadiness = when (availability) {
StandardVoiceAvailability.Ready ->
HermesVoiceActivationReadiness.Ready(HermesVoiceActivationRoute.Standard)
StandardVoiceAvailability.SignInRequired ->
HermesVoiceActivationReadiness.Blocked(
"Vanilla Hermes voice needs dashboard sign-in in Manage"
)
StandardVoiceAvailability.Unsupported ->
HermesVoiceActivationReadiness.Blocked(
"This Hermes dashboard does not expose the upstream audio routes"
)
StandardVoiceAvailability.Unreachable ->
HermesVoiceActivationReadiness.Waiting("Waiting for the Hermes dashboard")
StandardVoiceAvailability.Unknown ->
HermesVoiceActivationReadiness.Waiting("Checking Vanilla Hermes voice")
}
File diff suppressed because it is too large Load Diff
@@ -72,11 +72,13 @@ import androidx.compose.ui.text.input.ImeAction
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.compose.ui.res.stringResource
import com.hermesandroid.relay.data.ChatMessage
import com.hermesandroid.relay.data.MessageRole
import com.hermesandroid.relay.ui.components.avatar.AvatarRenderState
import com.hermesandroid.relay.ui.components.avatar.LocalAgentAvatar
import com.hermesandroid.relay.ui.components.avatar.LocalBackgroundVisualizationEnabled
import kotlinx.coroutines.delay
import com.hermesandroid.relay.R
import com.hermesandroid.relay.ui.theme.RelayRefresh
@@ -261,8 +263,12 @@ fun AgentTextFlow(
motionEnabled: Boolean,
modifier: Modifier = Modifier,
) {
val flowStyle = MaterialTheme.typography.bodyMedium.copy(fontFamily = FontFamily.Monospace)
val flowColor = MaterialTheme.colorScheme.onSurfaceVariant
val flowStyle = MaterialTheme.typography.bodyMedium.copy(
fontFamily = FontFamily.Monospace,
fontSize = 15.sp,
lineHeight = 21.sp,
)
val flowColor = MaterialTheme.colorScheme.onSurface
// Readable, non-faded mirror of the visible tail — used as the live-region
// text on both paths so assistive tech hears the words.
@@ -496,12 +502,9 @@ private fun CleanModeComposer(
* the caller as plain UI-local state; this is a presentation over the same
* conversation, not new ViewModel state.
*
* Honors [animationEnabled], OS reduce-motion, and TalkBack: the sphere
* renders a static frame and the text stays readable + announced when motion
* is suppressed.
*
* The avatar is rendered through the [LocalAgentAvatar] seam (WP-C2), so clean
* mode gets future "pets" for free alongside chat and the voice overlay.
* Honors the ambient-visualization preference independently from motion.
* When visible, [animationEnabled] and OS reduce-motion pause the sphere on a
* static frame; TalkBack keeps the text readable and announced.
*/
@Composable
fun CleanChatMode(
@@ -517,6 +520,7 @@ fun CleanChatMode(
modifier: Modifier = Modifier,
) {
val motion = rememberAccessibleMotionState()
val backgroundVisualizationEnabled = LocalBackgroundVisualizationEnabled.current
val sphereAnimated = animationEnabled && motion.osAnimations
// Faded text is unreadable to touch exploration, so the text path goes
// static (readable + announced) whenever TalkBack is exploring.
@@ -603,29 +607,32 @@ fun CleanChatMode(
// sphere rises toward the top third.
Spacer(modifier = Modifier.weight(1f))
// Bounded, centered sphere — a fixed size so the group grows via the
// text, sliding the sphere upward as the conversation lengthens.
Box(
modifier = Modifier
.fillMaxWidth()
.height(sphereHeight),
contentAlignment = Alignment.Center,
) {
// The ambient sphere is optional. Hiding it collapses the visual's
// slot so the conversation remains centered instead of leaving a
// large blank region above the text.
if (backgroundVisualizationEnabled) {
Box(
modifier = Modifier
.fillMaxSize()
.semantics { contentDescription = sphereDescription },
.fillMaxWidth()
.height(sphereHeight),
contentAlignment = Alignment.Center,
) {
LocalAgentAvatar.current.Render(
state = AvatarRenderState(
state = sphereState,
intensity = streamingIntensity,
toolCallBurst = toolCallBurst,
// Pin to a still frame when motion is suppressed.
paused = !sphereAnimated,
),
modifier = Modifier.fillMaxSize(),
)
Box(
modifier = Modifier
.fillMaxSize()
.semantics { contentDescription = sphereDescription },
) {
LocalAgentAvatar.current.Render(
state = AvatarRenderState(
state = sphereState,
intensity = streamingIntensity,
toolCallBurst = toolCallBurst,
// Pin to a still frame when motion is suppressed.
paused = !sphereAnimated,
),
modifier = Modifier.fillMaxSize(),
)
}
}
}
@@ -5,7 +5,9 @@ import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.core.tween
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.animation.expandVertically
import androidx.compose.animation.scaleIn
import androidx.compose.animation.shrinkVertically
import androidx.compose.animation.togetherWith
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.ExperimentalFoundationApi
@@ -130,6 +132,12 @@ data class ChatInputPickerControl(
* (DataStore flag owned by the caller, consumed via [onVoiceHintShown]).
* - [purpleGlow] on the trailing button (dark theme only) when it is an
* enabled SEND — the bar's one flourish, exactly as before.
* - [topContent] lets an active mode share the composer's outer surface.
* Conversation voice uses it for the dock and sets [suppressVoiceTrailing]
* so the dock remains the only idle/stop voice action; typed send/steer/
* queue actions still render normally.
* - [topContentVisible] expands or collapses that shared content from the
* composer edge instead of abruptly replacing the input layout.
*
* [onSend] fires for SEND, STEER, and QUEUE — the caller already encoded
* the meaning in the state it passed; [onVoice]/[onStop] for theirs.
@@ -160,6 +168,9 @@ fun ChatInputBar(
onModelPickerClick: (() -> Unit)? = null,
effortControl: ChatInputPickerControl? = null,
onEffortOptionSelected: (ChatInputPickerOption) -> Unit = {},
topContent: (@Composable () -> Unit)? = null,
topContentVisible: Boolean = topContent != null,
suppressVoiceTrailing: Boolean = false,
modifier: Modifier = Modifier,
enabled: Boolean = true,
) {
@@ -246,35 +257,57 @@ fun ChatInputBar(
.fillMaxWidth()
.padding(horizontal = 8.dp, vertical = 6.dp),
) {
Column(
modifier = Modifier.padding(horizontal = 8.dp, vertical = 6.dp),
) {
BasicTextField(
value = value,
onValueChange = { if (it.length <= charLimit) onValueChange(it) },
modifier = Modifier
.fillMaxWidth()
.heightIn(min = 34.dp)
.padding(horizontal = 8.dp, vertical = 4.dp),
maxLines = 5,
enabled = enabled,
textStyle = MaterialTheme.typography.bodyLarge.copy(
color = MaterialTheme.colorScheme.onSurface,
),
cursorBrush = SolidColor(MaterialTheme.colorScheme.primary),
decorationBox = { inner ->
Box(Modifier.fillMaxWidth()) {
if (value.isEmpty()) {
Text(
text = placeholder,
style = MaterialTheme.typography.bodyLarge,
color = RelayRefresh.Dim,
)
}
inner()
Column {
if (topContent != null) {
AnimatedVisibility(
visible = topContentVisible,
enter = expandVertically(
animationSpec = tween(240),
expandFrom = Alignment.Bottom,
) + fadeIn(tween(180)),
exit = shrinkVertically(
animationSpec = tween(180),
shrinkTowards = Alignment.Bottom,
) + fadeOut(tween(120)),
) {
Column {
topContent()
HorizontalDivider(
color = MaterialTheme.colorScheme.outlineVariant.copy(alpha = 0.72f),
)
}
},
)
}
}
Column(
modifier = Modifier.padding(horizontal = 8.dp, vertical = 6.dp),
) {
BasicTextField(
value = value,
onValueChange = { if (it.length <= charLimit) onValueChange(it) },
modifier = Modifier
.fillMaxWidth()
.heightIn(min = 34.dp)
.padding(horizontal = 8.dp, vertical = 4.dp),
maxLines = 5,
enabled = enabled,
textStyle = MaterialTheme.typography.bodyLarge.copy(
color = MaterialTheme.colorScheme.onSurface,
),
cursorBrush = SolidColor(MaterialTheme.colorScheme.primary),
decorationBox = { inner ->
Box(Modifier.fillMaxWidth()) {
if (value.isEmpty()) {
Text(
text = placeholder,
style = MaterialTheme.typography.bodyLarge,
color = RelayRefresh.Dim,
)
}
inner()
}
},
)
Row(
modifier = Modifier
@@ -401,42 +434,50 @@ fun ChatInputBar(
)
}
ChatInputTrailing.VOICE -> Box {
IconButton(onClick = onVoice) {
Icon(
imageVector = Icons.Filled.GraphicEq,
contentDescription = if (voiceReady) stringResource(R.string.chat_input_start_voice)
else stringResource(R.string.chat_input_voice_setup_needed),
tint = MaterialTheme.colorScheme.primary,
)
}
// "Needs setup" badge — full-alpha button + Amber
// dot instead of a half-dimmed broken-looking mic.
if (!voiceReady) {
Box(
modifier = Modifier
.align(Alignment.TopEnd)
.padding(top = 8.dp, end = 8.dp)
.size(6.dp)
.clip(CircleShape)
.background(RelayRefresh.Amber),
)
ChatInputTrailing.VOICE -> {
if (!suppressVoiceTrailing) {
Box {
IconButton(onClick = onVoice) {
Icon(
imageVector = Icons.Filled.GraphicEq,
contentDescription = if (voiceReady) stringResource(R.string.chat_input_start_voice)
else stringResource(R.string.chat_input_voice_setup_needed),
tint = MaterialTheme.colorScheme.primary,
)
}
// "Needs setup" badge — full-alpha button + Amber
// dot instead of a half-dimmed broken-looking mic.
if (!voiceReady) {
Box(
modifier = Modifier
.align(Alignment.TopEnd)
.padding(top = 8.dp, end = 8.dp)
.size(6.dp)
.clip(CircleShape)
.background(RelayRefresh.Amber),
)
}
}
}
}
ChatInputTrailing.STOP -> IconButton(onClick = onStop) {
Box(
modifier = Modifier
.size(32.dp)
.border(1.dp, MaterialTheme.colorScheme.error, CircleShape),
contentAlignment = Alignment.Center,
) {
Icon(
imageVector = Icons.Filled.Stop,
contentDescription = stringResource(R.string.chat_input_stop_streaming),
tint = MaterialTheme.colorScheme.error,
modifier = Modifier.size(18.dp),
)
ChatInputTrailing.STOP -> {
if (!suppressVoiceTrailing) {
IconButton(onClick = onStop) {
Box(
modifier = Modifier
.size(32.dp)
.border(1.dp, MaterialTheme.colorScheme.error, CircleShape),
contentAlignment = Alignment.Center,
) {
Icon(
imageVector = Icons.Filled.Stop,
contentDescription = stringResource(R.string.chat_input_stop_streaming),
tint = MaterialTheme.colorScheme.error,
modifier = Modifier.size(18.dp),
)
}
}
}
}
@@ -480,6 +521,7 @@ fun ChatInputBar(
}
}
}
}
@Composable
private fun ChatInputPickerChip(
File diff suppressed because it is too large Load Diff
@@ -63,6 +63,8 @@ fun DiagnosticDetailDialog(entry: DiagnosticLogEntry, onDismiss: () -> Unit) {
val context = LocalContext.current
val plainText = remember(entry) { entry.toPlainText() }
val severityName = entry.severity.name
val copiedToast = stringResource(R.string.diag_copied)
val exportChooserTitle = stringResource(R.string.diag_export)
// Info-severity pre-flight: routine log lines only become GitHub issues once
// the reporter says what they expected instead (that answer replaces the
@@ -165,7 +167,7 @@ fun DiagnosticDetailDialog(entry: DiagnosticLogEntry, onDismiss: () -> Unit) {
OutlinedButton(
onClick = {
IssueReport.copyToClipboard(context, plainText)
toast(context, "Diagnostic copied")
toast(context, copiedToast)
},
) { Text(stringResource(R.string.common_copy)) }
OutlinedButton(
@@ -174,7 +176,7 @@ fun DiagnosticDetailDialog(entry: DiagnosticLogEntry, onDismiss: () -> Unit) {
context,
subject = "Hermes-Relay diagnostic — ${entry.title}",
text = plainText,
chooserTitle = "Export diagnostic",
chooserTitle = exportChooserTitle,
)
if (!shared) {
IssueReport.copyToClipboard(context, plainText)
@@ -246,7 +248,7 @@ internal fun DiagnosticSeverityChip(severity: DiagnosticSeverity) {
}
Surface(shape = RoundedCornerShape(50), color = bg) {
Text(
text = severity.name.uppercase(),
text = stringResource(severityLabelRes(severity)).uppercase(),
style = MaterialTheme.typography.labelSmall,
fontWeight = FontWeight.SemiBold,
color = fg,
@@ -99,7 +99,7 @@ fun DiagnosticsLogPanel(
FilterChip(
selected = severityFilter == sev,
onClick = { severityFilter = if (severityFilter == sev) null else sev },
label = { Text(sev.name) },
label = { Text(stringResource(severityLabelRes(sev))) },
)
}
}
@@ -224,3 +224,10 @@ private fun DiagnosticLogEntry.detailLine(): String? {
)
return pieces.joinToString(" - ").takeIf { it.isNotBlank() }
}
/** String resource for a [DiagnosticSeverity] label (shared with the detail dialog). */
internal fun severityLabelRes(severity: DiagnosticSeverity): Int = when (severity) {
DiagnosticSeverity.Info -> R.string.diag_severity_info
DiagnosticSeverity.Warning -> R.string.diag_severity_warning
DiagnosticSeverity.Error -> R.string.diag_severity_error
}
@@ -78,9 +78,9 @@ fun GatewayBackgroundProcessStrip(
val failed = processes.count { !it.isRunning && (it.exitCode ?: 0) != 0 }
val displayedCount = if (running > 0) running else processes.size
val status = when {
running > 0 -> "$running running"
failed > 0 -> "$failed failed"
else -> "Complete"
running > 0 -> "$running ${stringResource(R.string.bg_processes_running)}"
failed > 0 -> "$failed ${stringResource(R.string.task_status_failed)}"
else -> stringResource(R.string.task_status_complete)
}
Surface(
@@ -94,7 +94,7 @@ fun GatewayBackgroundProcessStrip(
stateDescription = status
}
.clickable(
onClickLabel = "Open background processes",
onClickLabel = stringResource(R.string.bg_processes_open),
onClick = onClick,
),
shape = RoundedCornerShape(14.dp),
@@ -208,7 +208,7 @@ fun GatewayBackgroundProcessSheet(
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
Text(
"No background processes in this chat",
stringResource(R.string.bg_processes_empty),
modifier = Modifier.padding(top = 12.dp),
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
@@ -220,7 +220,7 @@ fun GatewayBackgroundProcessSheet(
.heightIn(max = 560.dp),
) {
if (running.isNotEmpty()) {
item { ProcessSectionLabel("Running", running.size) }
item { ProcessSectionLabel(stringResource(R.string.bg_processes_running), running.size) }
items(running, key = { it.id }) { process ->
GatewayProcessRow(
process = process,
@@ -234,7 +234,7 @@ fun GatewayBackgroundProcessSheet(
item { HorizontalDivider(modifier = Modifier.padding(vertical = 6.dp)) }
}
if (recent.isNotEmpty()) {
item { ProcessSectionLabel("Recent", recent.size) }
item { ProcessSectionLabel(stringResource(R.string.bg_processes_recent), recent.size) }
items(recent, key = { it.id }) { process ->
GatewayProcessRow(
process = process,
@@ -278,7 +278,7 @@ private fun GatewayProcessRow(
?.trim()
.orEmpty()
.ifBlank {
"Background process"
stringResource(R.string.bg_processes_title)
}
Column(
@@ -287,7 +287,7 @@ private fun GatewayProcessRow(
.animateContentSize()
.clickable(
enabled = output.isNotBlank(),
onClickLabel = if (expanded) "Collapse process output" else "Expand process output",
onClickLabel = if (expanded) stringResource(R.string.bg_collapse_output) else stringResource(R.string.bg_expand_output),
) { expanded = !expanded }
.padding(horizontal = 20.dp, vertical = 10.dp),
) {
@@ -332,7 +332,7 @@ private fun GatewayProcessRow(
if (output.isNotBlank()) {
Icon(
imageVector = if (expanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore,
contentDescription = if (expanded) "Collapse output" else "Expand output",
contentDescription = if (expanded) stringResource(R.string.tool_progress_cd_collapse) else stringResource(R.string.tool_progress_cd_expand),
modifier = Modifier.size(20.dp),
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
@@ -396,11 +396,14 @@ private fun ProcessStateIcon(process: GatewayProcess, failed: Boolean, stopping:
}
}
@Composable
private fun processMetadata(process: GatewayProcess, failed: Boolean): String {
val state = when {
process.isRunning -> "Running"
failed -> "Failed${process.exitCode?.let { " · exit $it" }.orEmpty()}"
else -> "Completed${process.exitCode?.let { " · exit $it" }.orEmpty()}"
process.isRunning -> stringResource(R.string.bg_processes_running)
failed -> stringResource(R.string.task_status_failed) +
process.exitCode?.let { " · exit $it" }.orEmpty()
else -> stringResource(R.string.task_status_complete) +
process.exitCode?.let { " · exit $it" }.orEmpty()
}
return "$state · ${formatElapsed(process.uptimeSeconds)}" +
if (process.detached) " · recovered" else ""
@@ -502,7 +502,7 @@ private fun CardInputSlot(
input.holdToConfirm -> {
Spacer(Modifier.height(10.dp))
HoldToConfirmButton(
label = "Hold to confirm",
label = stringResource(R.string.hermes_card_hold_confirm),
enabled = !input.masked || answerText.isNotEmpty(),
onConfirmed = {
onSubmit(
@@ -511,6 +511,13 @@ private fun CardInputSlot(
)
},
)
Spacer(Modifier.height(4.dp))
Text(
text = stringResource(R.string.hermes_card_hold_hint),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f),
modifier = Modifier.fillMaxWidth(),
)
}
input.masked -> {
Spacer(Modifier.height(10.dp))
@@ -155,7 +155,7 @@ private fun LoadingCard(
val sizeHint = attachment.fileSize?.takeIf { it > 0 }
?.let { " · ${formatBytes(it)}" } ?: ""
Text(
text = if (isManualCta) stringResource(R.string.inbound_attach_tap_download) else stringResource(R.string.inbound_attach_downloading, sizeHint),
text = if (isManualCta) stringResource(R.string.attach_tap_download) else stringResource(R.string.inbound_attach_downloading, sizeHint),
style = MaterialTheme.typography.bodyMedium,
fontWeight = FontWeight.Medium,
color = MaterialTheme.colorScheme.onSurfaceVariant
@@ -0,0 +1,117 @@
package com.hermesandroid.relay.ui.components
import androidx.compose.runtime.Composable
import androidx.compose.ui.res.stringResource
import com.hermesandroid.relay.R
/**
* Returns the string resource for a tool name (toolCall.name), or null when the
* name does not map to a known tool. Pure mapping so it can be unit-tested.
*/
fun localizeToolNameKey(name: String): Int? = when {
name.contains("terminal") -> R.string.tool_name_terminal
name.contains("execute") -> R.string.tool_name_execute_code
name.contains("read_file") -> R.string.tool_name_read_file
name.contains("write_file") || name.contains("patch") -> R.string.tool_name_write_file
name.contains("session_search") -> R.string.tool_name_session
name.contains("android") || name.contains("phone") -> R.string.tool_name_android
name.contains("web_search") || name.contains("search") -> R.string.tool_name_web_search
name.contains("web_extract") -> R.string.tool_name_web_extract
name.contains("memory") || name.contains("mnemosyne") -> R.string.tool_name_memory
name.contains("skill") -> R.string.tool_name_skill
name.contains("delegate") -> R.string.tool_name_delegate
name.contains("cron") -> R.string.tool_name_cron
name.contains("todo") -> R.string.tool_name_todo
name.contains("process") -> R.string.tool_name_process
name.contains("vision") || name.contains("image") -> R.string.tool_name_vision
name.contains("computer_use") -> R.string.tool_name_computer
name.contains("speech") || name.contains("tts") -> R.string.tool_name_tts
name.contains("file") -> R.string.tool_name_file
else -> null
}
/**
* Returns the string resource for a badge label (message.badges entries), or
* null when the badge does not map to a known resource. Pure mapping so it can
* be unit-tested.
*/
fun localizeBadgeKey(badge: String): Int? = when (badge) {
"Tool failed" -> R.string.badge_tool_failed
"Memory" -> R.string.badge_memory
"Skill" -> R.string.badge_skill
"Artifact" -> R.string.badge_artifact
"Response interrupted" -> R.string.badge_response_interrupted
"Unknown error" -> R.string.badge_unknown_error
"Error" -> R.string.badge_error
"Stopped" -> R.string.badge_stopped
"Model changed" -> R.string.badge_model_changed
"Background work completed" -> R.string.badge_bg_work_completed
"Continued after an interrupted turn" -> R.string.badge_continued
"Realtime Agent" -> R.string.bubble_realtime_agent
"Voice" -> R.string.bubble_voice
else -> null
}
/**
* Returns the string resource for an agent name (message.agentName), or null
* when the name does not map to a known resource. Pure mapping so it can be
* unit-tested.
*/
fun localizeAgentNameKey(name: String): Int? = when (name) {
"Send SMS" -> R.string.agent_send_sms
"Call" -> R.string.agent_call
"Search Contacts" -> R.string.agent_search_contacts
"Open App" -> R.string.agent_open_app
"Return to Hermes" -> R.string.agent_return_hermes
"Screenshot" -> R.string.agent_screenshot
"Key Press" -> R.string.agent_key_press
"Bridge Setup" -> R.string.agent_bridge_setup
else -> null
}
/** Localizes a tool name (toolCall.name) for display. Unknown names are returned as-is. */
@Composable
fun localizeToolName(name: String): String = localizeToolNameKey(name)?.let {
stringResource(it)
} ?: name
/** Localizes a badge (message.badges entries) for display. Unknown badges are returned as-is. */
@Composable
fun localizeBadge(badge: String): String = localizeBadgeKey(badge)?.let {
stringResource(it)
} ?: badge
/** Localizes an agent name (message.agentName) for display. Unknown names are returned as-is. */
@Composable
fun localizeAgentName(name: String): String = localizeAgentNameKey(name)?.let {
stringResource(it)
} ?: name
/**
* Splits a timeline event title of the form "toolName · status" and returns the
* resource key for the tool-name segment when it is known. Returns null when
* the title has no separator or the name is not mapped, so the render site
* falls back to the original title.
*/
fun localizeTimelineTitleName(title: String): Pair<String, Int?>? {
val sep = " · "
val idx = title.indexOf(sep)
if (idx <= 0) return null
val name = title.substring(0, idx)
return name to localizeToolNameKey(name)
}
/**
* Localizes a timeline event title of the form "toolName · status" at the
* render site (TimelineRow). The data layer (buildTimelineEvents) keeps the
* English keys.
*/
@Composable
fun localizeTimelineTitle(title: String): String {
val parts = localizeTimelineTitleName(title) ?: return title
val (name, key) = parts
if (key == null) return title
val sep = " · "
val idx = title.indexOf(sep)
return stringResource(key) + title.substring(idx)
}
@@ -67,6 +67,11 @@ fun MarkdownContent(
modifier: Modifier = Modifier
) {
val isDarkTheme = LocalBrand.current.isDark
val chatBodyStyle = MaterialTheme.typography.bodyMedium.copy(
fontSize = 15.sp,
lineHeight = 21.sp,
color = textColor,
)
val highlightsBuilder = remember(isDarkTheme) {
Highlights.Builder().theme(SyntaxThemes.atom(darkMode = isDarkTheme))
}
@@ -87,7 +92,8 @@ fun MarkdownContent(
// ~45sp, h3=displaySmall 36sp) — a single `#` becomes a billboard inside the
// ~272dp bubble. Here every level derives from bodyLarge/bodyMedium (so the
// live font-picker still applies) and is capped so the largest heading is
// ~1.4x the 14sp body, matching Discord / GitHub-mobile in-message headings.
// proportionate to the 15sp body, matching Discord / GitHub-mobile
// in-message headings.
typography = markdownTypography(
h1 = MaterialTheme.typography.bodyLarge.copy(
fontSize = 20.sp, lineHeight = 26.sp, fontWeight = FontWeight.Bold, color = textColor,
@@ -108,16 +114,17 @@ fun MarkdownContent(
fontSize = 13.sp, fontWeight = FontWeight.SemiBold, letterSpacing = 0.4.sp,
color = textColor.copy(alpha = 0.85f),
),
// Prose, list items, and quotes all sit at the 14sp body size so a
// paragraph and the bullet list under it share one rhythm — the library
// default 'text'/list role is bodyLarge (16sp), 2sp larger than paragraph.
paragraph = MaterialTheme.typography.bodyMedium.copy(color = textColor),
text = MaterialTheme.typography.bodyMedium.copy(color = textColor),
bullet = MaterialTheme.typography.bodyMedium.copy(color = textColor),
ordered = MaterialTheme.typography.bodyMedium.copy(color = textColor),
list = MaterialTheme.typography.bodyMedium.copy(color = textColor),
quote = MaterialTheme.typography.bodyMedium.copy(
fontStyle = FontStyle.Italic, color = textColor.copy(alpha = 0.78f),
// Prose, list items, and quotes share a 15sp/21sp reading rhythm.
// The library default 'text'/list role is bodyLarge (16sp), while
// bodyMedium was previously 14sp and unnecessarily small for long chat.
paragraph = chatBodyStyle,
text = chatBodyStyle,
bullet = chatBodyStyle,
ordered = chatBodyStyle,
list = chatBodyStyle,
quote = chatBodyStyle.copy(
fontStyle = FontStyle.Italic,
color = textColor.copy(alpha = 0.9f),
),
// Inline + fenced code at 13sp (one step under body, not two): monospace
// + the tinted chip already signal "code" without also shrinking it, and
@@ -125,7 +132,7 @@ fun MarkdownContent(
code = MaterialTheme.typography.bodySmall.copy(
fontSize = 13.sp, letterSpacing = 0.sp,
fontFamily = FontFamily.Monospace,
color = MaterialTheme.colorScheme.onSurfaceVariant,
color = textColor,
),
inlineCode = MaterialTheme.typography.bodyMedium.copy(
fontSize = 13.sp, letterSpacing = 0.sp,
@@ -302,7 +309,10 @@ fun StreamingMarkdownContent(
// code and deliberately spaced prose are not altered.
text = content.withoutLeadingBlankLines(),
modifier = modifier,
style = MaterialTheme.typography.bodyMedium,
style = MaterialTheme.typography.bodyMedium.copy(
fontSize = 15.sp,
lineHeight = 21.sp,
),
color = textColor,
)
} else {
@@ -29,8 +29,9 @@ import androidx.compose.foundation.shape.CircleShape
import androidx.compose.ui.draw.clip
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.text.selection.SelectionContainer
import androidx.compose.foundation.text.selection.DisableSelection
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.VolumeUp
import androidx.compose.material.icons.automirrored.filled.VolumeUp
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
@@ -50,7 +51,6 @@ import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.vector.ImageVector
import androidx.compose.ui.hapticfeedback.HapticFeedbackType
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.platform.LocalHapticFeedback
import androidx.compose.ui.platform.LocalLocale
@@ -62,7 +62,6 @@ import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import coil3.compose.AsyncImage
import com.hermesandroid.relay.R
import com.hermesandroid.relay.data.BlurMode
import com.hermesandroid.relay.data.ChatMessage
@@ -70,9 +69,10 @@ import com.hermesandroid.relay.data.HermesCardAction
import com.hermesandroid.relay.data.MediaSettingsRepository
import com.hermesandroid.relay.data.MessageDeliveryStatus
import com.hermesandroid.relay.data.MessageRole
import com.hermesandroid.relay.ui.components.pet.petPerchSurface
import com.hermesandroid.relay.ui.components.pet.petVisitTargetSurface
import com.hermesandroid.relay.ui.theme.leftEdgeGlow
import kotlinx.coroutines.delay
import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
@@ -97,6 +97,12 @@ fun MessageBubble(
* the long-press menu, so legacy call sites keep the copy-only behavior.
*/
onQuoteMessage: ((String) -> Unit)? = null,
/**
* Reads a completed assistant response through the active voice renderer.
* Null hides the entry; the owning screen uses that to limit the action to
* idle Conversation voice sessions.
*/
onSpeakMessage: ((String) -> Unit)? = null,
/**
* Invoked when the user taps a FAILED inbound attachment card.
* `attachmentIndex` is the position in [ChatMessage.attachments] so the
@@ -140,6 +146,8 @@ fun MessageBubble(
recoveringAnswer: Boolean = false,
imageGenerationStylePreference: String = "rotate",
imageGenerationRotationIndex: Int = 0,
petVisitTargetKey: String? = null,
petPerchKey: String? = null,
) {
val isUser = message.role == MessageRole.USER
val isSystem = message.role == MessageRole.SYSTEM
@@ -174,7 +182,7 @@ fun MessageBubble(
val textColor = when (message.role) {
MessageRole.USER -> MaterialTheme.colorScheme.onPrimary
MessageRole.ASSISTANT -> MaterialTheme.colorScheme.onSurfaceVariant
MessageRole.ASSISTANT -> MaterialTheme.colorScheme.onSurface
MessageRole.SYSTEM -> MaterialTheme.colorScheme.onTertiaryContainer
}
@@ -240,38 +248,54 @@ fun MessageBubble(
val blurMode by blurRepo.blurMode.collectAsState(initial = BlurMode.FLAGGED)
CompositionLocalProvider(LocalMediaBlurMode provides blurMode) {
// Identity (the active profile avatar) is shown once in the top bar, so
// message bubbles no longer reserve a per-group avatar gutter — that width
// is reclaimed for wider bubbles. Outer alignment keeps user bubbles right.
// The active profile image is sender identity, distinct from both the
// floating pet companion and the ambient Sphere. Reserve one stable gutter
// for an assistant run and render the identity only on its first message.
Row(
modifier = modifier.fillMaxWidth(),
verticalAlignment = Alignment.Top,
) {
if (!isUser && !isSystem) {
Box(
modifier = Modifier.width(40.dp),
contentAlignment = Alignment.TopCenter,
) {
if (shouldShowMessageGroupAvatar(
isUser = isUser,
isSystem = isSystem,
isFirstInGroup = isFirstInGroup,
agentName = message.agentName,
)
) {
Surface(
// Decorative: the adjacent visible agent name already owns
// the identity announcement, avoiding duplicate TalkBack copy.
modifier = Modifier.size(32.dp),
shape = CircleShape,
color = MaterialTheme.colorScheme.primary,
) {
AgentAvatarFace(
name = message.agentName.orEmpty(),
letterStyle = MaterialTheme.typography.labelMedium,
)
}
}
}
}
Column(
modifier = Modifier.fillMaxWidth(),
modifier = Modifier.weight(1f),
horizontalAlignment = alignment
) {
// Agent name label (above assistant bubbles, only first in group), with
// the active profile's local icon (if set) — a small avatar by the name.
// Agent name label sits beside the first group identity avatar. Pet
// companions never enter this row.
if (!isUser && !isSystem && isFirstInGroup && !message.agentName.isNullOrBlank()) {
val agentIconPath = LocalAgentIconPath.current
Row(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(4.dp),
modifier = Modifier.padding(bottom = 2.dp, start = 4.dp),
) {
if (!agentIconPath.isNullOrBlank()) {
AsyncImage(
model = File(agentIconPath),
contentDescription = null,
contentScale = ContentScale.Crop,
modifier = Modifier
.size(16.dp)
.clip(CircleShape),
)
}
Text(
text = message.agentName,
text = localizeAgentName(message.agentName),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.primary,
)
@@ -288,13 +312,13 @@ fun MessageBubble(
) {
message.badges.take(4).forEach { badge ->
MessagePathBadge(
text = badge,
text = localizeBadge(badge),
// Speaker glyph = the shared "spoken" modality marker.
// Both the standard voice-mode chip ("Voice") and the
// realtime engine chip ("Realtime Agent") are spoken
// turns, so they share it; only the text differs.
leadingIcon = if (badge == "Voice" || badge == "Realtime Agent") {
Icons.Filled.VolumeUp
Icons.AutoMirrored.Filled.VolumeUp
} else {
null
},
@@ -309,6 +333,7 @@ fun MessageBubble(
thinkingContent = message.thinkingContent,
isStreaming = message.isThinkingStreaming,
timestamp = message.timestamp,
petObstacleKey = "chat-thinking:${message.uiKey}",
modifier = Modifier
.widthIn(max = maxBubbleWidth)
.padding(bottom = 4.dp)
@@ -321,17 +346,18 @@ fun MessageBubble(
thinkingContent = if (reference.available) {
reference.text
} else {
"Advisor unavailable."
stringResource(R.string.bubble_advisor_unavailable)
},
isStreaming = false,
petObstacleKey = "chat-thinking:${message.uiKey}:advisor:${reference.index}",
headerText = buildString {
append("Advisor ")
append(stringResource(R.string.bubble_advisor_prefix))
append(reference.index)
reference.count?.let { append("/").append(it) }
append(" · ")
append(reference.label)
},
accessibilityLabel = "Mixture of Agents advisor response",
accessibilityLabel = stringResource(R.string.bubble_moa_advisor),
modifier = Modifier
.widthIn(max = maxBubbleWidth)
.padding(bottom = 4.dp),
@@ -382,7 +408,8 @@ fun MessageBubble(
var showMessageActions by remember { mutableStateOf(false) }
val haptic = LocalHapticFeedback.current
val showEditAction = onEditMessage != null && isUser
if (onQuoteMessage != null || showEditAction) {
val showSpeakAction = shouldShowSpeakResponseAction(message, onSpeakMessage != null)
if (onQuoteMessage != null || showEditAction || showSpeakAction) {
DropdownMenu(
expanded = showMessageActions,
onDismissRequest = { showMessageActions = false },
@@ -403,6 +430,21 @@ fun MessageBubble(
},
)
}
if (showSpeakAction) {
DropdownMenuItem(
text = { Text(stringResource(R.string.msg_bubble_speak_response)) },
leadingIcon = {
Icon(
imageVector = Icons.AutoMirrored.Filled.VolumeUp,
contentDescription = null,
)
},
onClick = {
showMessageActions = false
onSpeakMessage?.invoke(message.content)
},
)
}
if (showEditAction) {
DropdownMenuItem(
text = { Text(stringResource(R.string.msg_bubble_edit)) },
@@ -454,7 +496,7 @@ fun MessageBubble(
// action menu is the discoverability moment, so it gets the
// same tactile confirm every chat app fires.
haptic.performHapticFeedback(HapticFeedbackType.LongPress)
if (onQuoteMessage != null || showEditAction) {
if (onQuoteMessage != null || showEditAction || showSpeakAction) {
showMessageActions = true
} else {
onCopyMessage(message.content)
@@ -462,14 +504,37 @@ fun MessageBubble(
}
)
.semantics { contentDescription = a11yDescription }
.then(
if (petVisitTargetKey != null) {
Modifier.petVisitTargetSurface(
key = petVisitTargetKey,
routes = setOf("chat"),
)
} else {
Modifier
}
)
.then(
if (petPerchKey != null) {
Modifier.petPerchSurface(
key = petPerchKey,
routes = setOf("chat"),
)
} else {
Modifier
}
)
) {
Column(modifier = Modifier.padding(horizontal = 14.dp, vertical = 9.dp)) {
SelectionContainer {
val messageTextContent: @Composable () -> Unit = {
if (isUser || isSystem) {
// Plain text for user and system messages
Text(
text = message.content,
style = MaterialTheme.typography.bodyMedium,
style = MaterialTheme.typography.bodyMedium.copy(
fontSize = 15.sp,
lineHeight = 21.sp,
),
color = textColor
)
} else {
@@ -487,6 +552,15 @@ fun MessageBubble(
}
}
}
// Conversation voice owns long-press with its action menu.
// Disable partial-text selection in that state so Android's
// floating selection toolbar does not stack over Copy/Quote/
// Speak response. Normal chat retains selectable message text.
if (showSpeakAction) {
DisableSelection { messageTextContent() }
} else {
SelectionContainer { messageTextContent() }
}
// Inline generated images (assistant only) — rendered OUTSIDE
// the SelectionContainer (they're not selectable text). Remote
@@ -717,6 +791,69 @@ fun MessageBubble(
} // end CompositionLocalProvider(LocalMediaBlurMode)
}
/**
* Only a settled, plain assistant response can become a transient pet visit
* target. Rich cards, attachments, and tool/action rows remain interaction
* surfaces for the user rather than pet destinations.
*/
internal fun isPetVisitTargetCandidate(message: ChatMessage): Boolean =
message.role == MessageRole.ASSISTANT &&
!message.isStreaming &&
!message.isThinkingStreaming &&
message.content.isNotBlank() &&
message.toolCalls.isEmpty() &&
message.cards.isEmpty() &&
message.attachments.isEmpty() &&
message.backgroundTask == null &&
message.agentName != "Voice action" &&
message.agentName != "Phone action" &&
!message.id.startsWith("voice-intent-")
/** The newest settled user or assistant bubble can provide text-safe habitat geometry. */
internal fun isPetPerchCandidate(message: ChatMessage): Boolean =
(message.role == MessageRole.ASSISTANT || message.role == MessageRole.USER) &&
!message.isStreaming &&
!message.isThinkingStreaming &&
message.content.isNotBlank() &&
message.backgroundTask == null &&
message.agentName != "Voice action" &&
message.agentName != "Phone action" &&
!message.id.startsWith("voice-intent-")
internal fun newestPetVisitTargetUiKey(messages: List<ChatMessage>): String? =
messages.lastOrNull { message ->
message.role == MessageRole.ASSISTANT
}?.takeIf(::isPetVisitTargetCandidate)?.uiKey
internal fun newestPetPerchUiKey(messages: List<ChatMessage>): String? =
messages.lastOrNull()?.takeIf(::isPetPerchCandidate)?.uiKey
/** All settled visible-message candidates that may become journey stepping stones. */
internal fun petPerchUiKeys(messages: List<ChatMessage>): Set<String> =
messages.asSequence().filter(::isPetPerchCandidate).map { it.uiKey }.toSet()
/** The completion edge is usable only after the newest assistant row itself settles. */
internal fun newestPetAssistantIsSettled(messages: List<ChatMessage>): Boolean =
messages.lastOrNull { it.role == MessageRole.ASSISTANT }?.let { message ->
!message.isStreaming && !message.isThinkingStreaming
} == true
internal fun shouldShowSpeakResponseAction(
message: ChatMessage,
handlerAvailable: Boolean,
): Boolean =
handlerAvailable &&
message.role == MessageRole.ASSISTANT &&
!message.isStreaming &&
message.content.isNotBlank()
internal fun shouldShowMessageGroupAvatar(
isUser: Boolean,
isSystem: Boolean,
isFirstInGroup: Boolean,
agentName: String?,
): Boolean = !isUser && !isSystem && isFirstInGroup && !agentName.isNullOrBlank()
@Composable
private fun MessagePathBadge(text: String, leadingIcon: ImageVector? = null) {
Surface(
@@ -83,8 +83,9 @@ fun ModelPickerSheet(
}
// groupBy preserves key insertion order, so providers stay current-first
// (the caller sorts is_current to the front).
val grouped = remember(filtered) {
filtered.groupBy { it.group?.takeIf { g -> g.isNotBlank() } ?: "Other" }
val otherGroupLabel = stringResource(R.string.model_picker_other)
val grouped = remember(filtered, otherGroupLabel) {
filtered.groupBy { it.group?.takeIf { g -> g.isNotBlank() } ?: otherGroupLabel }
}
ModalBottomSheet(
@@ -106,7 +107,7 @@ fun ModelPickerSheet(
verticalAlignment = Alignment.CenterVertically,
) {
Column(modifier = Modifier.weight(1f)) {
Text(text = "Model", style = MaterialTheme.typography.titleMedium)
Text(text = stringResource(R.string.model_picker_model), style = MaterialTheme.typography.titleMedium)
Text(
text = "${modelOptions.size} models",
style = MaterialTheme.typography.labelMedium,
@@ -131,7 +132,7 @@ fun ModelPickerSheet(
)
}
Spacer(modifier = Modifier.size(6.dp))
Text(if (refreshing) "Refreshing" else "Refresh")
Text(if (refreshing) stringResource(R.string.model_picker_refreshing) else stringResource(R.string.model_picker_refresh))
}
}
}
@@ -216,6 +217,63 @@ fun ModelPickerSheet(
}
}
/**
* Compact sibling of [ModelPickerSheet] for short, ungrouped control lists.
* Passport selectors use this so profile, personality, and reasoning choices
* share the model picker's modal hierarchy, row rhythm, and selected-state
* treatment instead of expanding dense radio lists inside the detail card.
*/
@OptIn(ExperimentalMaterial3Api::class)
@Composable
internal fun OptionPickerSheet(
title: String,
subtitle: String? = null,
options: List<ChatInputPickerOption>,
onSelect: (ChatInputPickerOption) -> Unit,
onDismiss: () -> Unit,
) {
val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true)
ModalBottomSheet(
onDismissRequest = onDismiss,
sheetState = sheetState,
shape = RoundedCornerShape(topStart = 20.dp, topEnd = 20.dp),
) {
Column(
modifier = Modifier
.fillMaxWidth()
.padding(bottom = 16.dp),
) {
Column(
modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp),
) {
Text(text = title, style = MaterialTheme.typography.titleMedium)
if (!subtitle.isNullOrBlank()) {
Text(
text = subtitle,
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
Spacer(modifier = Modifier.height(8.dp))
HorizontalDivider()
LazyColumn(
modifier = Modifier
.fillMaxWidth()
.heightIn(max = 520.dp),
) {
items(
items = options,
key = { option -> "${option.group}:${option.value}:${option.label}" },
) { option ->
ModelPickerRow(option = option, onClick = { onSelect(option) })
}
}
}
}
}
@Composable
private fun ModelPickerRow(
option: ChatInputPickerOption,
@@ -202,8 +202,8 @@ fun PowerFeatureGateCard(
private fun PowerFeatureGatePreview() {
HermesRelayTheme {
PowerFeatureGateCard(
title = "Terminal",
summary = "Open a server shell through your paired relay session.",
title = stringResource(R.string.power_terminal),
summary = stringResource(R.string.power_terminal_desc),
status = PowerFeatureGateStatus.RequiresPairing,
onPrimaryAction = {},
)
@@ -215,8 +215,8 @@ private fun PowerFeatureGatePreview() {
private fun PowerFeatureGateExpiredPreview() {
HermesRelayTheme {
PowerFeatureGateCard(
title = "Bridge",
summary = "Let Hermes send approved bridge commands to this phone.",
title = stringResource(R.string.power_bridge),
summary = stringResource(R.string.power_bridge_desc),
status = PowerFeatureGateStatus.PairingExpired,
onPrimaryAction = {},
)
@@ -25,6 +25,7 @@ import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Add
import androidx.compose.material.icons.filled.Archive
import androidx.compose.material.icons.filled.Check
import androidx.compose.material.icons.filled.ContentCopy
import androidx.compose.material.icons.filled.FilterList
import androidx.compose.material.icons.filled.Delete
import androidx.compose.material.icons.filled.Edit
@@ -90,6 +91,7 @@ fun SessionDrawerContent(
onSelectSession: (String) -> Unit,
onDeleteSession: (String) -> Unit,
onRenameSession: (String, String) -> Unit,
onCopySessionId: ((String) -> Unit)? = null,
/**
* When true, the Threads affordance (header spool + filter chip) shows even with no
* Thread sessions present yet — i.e. the relay Threads capability is paired + opted in
@@ -483,6 +485,7 @@ fun SessionDrawerContent(
}
},
onRename = { renameDialogSession = session },
onCopySessionId = { onCopySessionId?.invoke(session.sessionId) },
onDelete = { deleteDialogSession = session }
)
}
@@ -593,6 +596,7 @@ private fun SessionItem(
onTogglePinned: () -> Unit,
onToggleArchived: () -> Unit,
onRename: () -> Unit,
onCopySessionId: () -> Unit,
onDelete: () -> Unit
) {
var menuOpen by remember { mutableStateOf(false) }
@@ -711,6 +715,16 @@ private fun SessionItem(
expanded = menuOpen,
onDismissRequest = { menuOpen = false },
) {
DropdownMenuItem(
text = { Text(stringResource(R.string.chat_copy_session_id)) },
leadingIcon = {
Icon(Icons.Filled.ContentCopy, contentDescription = null)
},
onClick = {
menuOpen = false
onCopySessionId()
},
)
DropdownMenuItem(
text = { Text(stringResource(R.string.drawer_rename)) },
leadingIcon = {
@@ -28,9 +28,14 @@ import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalLocale
import androidx.compose.ui.res.stringResource
import com.hermesandroid.relay.R
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.hermesandroid.relay.ui.components.pet.petObstacleSurface
private val THINKING_BLOCK_PET_ROUTES = setOf("chat")
@Composable
fun ThinkingBlock(
@@ -40,7 +45,9 @@ fun ThinkingBlock(
/** Message timestamp shown right-aligned in the header (null hides it). */
timestamp: Long? = null,
headerText: String? = null,
accessibilityLabel: String = "Thinking",
accessibilityLabel: String = stringResource(R.string.thinking_thinking_short),
/** Stable key when this interactive block participates in Chat pet terrain. */
petObstacleKey: String? = null,
) {
var expanded by remember { mutableStateOf(isStreaming) }
val locale = LocalLocale.current.platformLocale
@@ -52,7 +59,18 @@ fun ThinkingBlock(
}
Card(
modifier = modifier.fillMaxWidth(),
modifier = modifier
.then(
if (petObstacleKey != null) {
Modifier.petObstacleSurface(
key = petObstacleKey,
routes = THINKING_BLOCK_PET_ROUTES,
)
} else {
Modifier
},
)
.fillMaxWidth(),
colors = CardDefaults.cardColors(
containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)
)
@@ -73,7 +91,7 @@ fun ThinkingBlock(
)
Spacer(modifier = Modifier.width(6.dp))
Text(
text = headerText ?: if (isStreaming) "Thinking..." else "Thought process",
text = headerText ?: if (isStreaming) stringResource(R.string.thinking_thinking) else stringResource(R.string.thinking_title),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.tertiary
)
@@ -88,7 +106,7 @@ fun ThinkingBlock(
}
Icon(
imageVector = if (expanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore,
contentDescription = if (expanded) "Collapse" else "Expand",
contentDescription = if (expanded) stringResource(R.string.tool_progress_cd_collapse) else stringResource(R.string.tool_progress_cd_expand),
modifier = Modifier.size(16.dp),
tint = MaterialTheme.colorScheme.onSurfaceVariant
)
@@ -38,6 +38,7 @@ import androidx.compose.ui.draw.clip
import androidx.compose.ui.draw.drawBehind
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.res.pluralStringResource
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
@@ -102,7 +103,7 @@ fun TimelineView(
color = MaterialTheme.colorScheme.onSurface,
)
Text(
text = "${events.size} events",
text = pluralStringResource(R.plurals.timeline_events_count, events.size, events.size),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
@@ -161,11 +162,11 @@ private fun TimelineLegend() {
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
LegendEntry("Chat", TimelineEventKind.ChatMessage.color())
LegendEntry("Tool", TimelineEventKind.ToolCall.color())
LegendEntry("Voice", TimelineEventKind.VoiceTurn.color())
LegendEntry("Profile", TimelineEventKind.ProfileSwitch.color())
LegendEntry("Conn", TimelineEventKind.ConnectionEvent.color())
LegendEntry(stringResource(R.string.timeline_legend_chat), TimelineEventKind.ChatMessage.color())
LegendEntry(stringResource(R.string.timeline_legend_tool), TimelineEventKind.ToolCall.color())
LegendEntry(stringResource(R.string.timeline_legend_voice), TimelineEventKind.VoiceTurn.color())
LegendEntry(stringResource(R.string.timeline_legend_profile), TimelineEventKind.ProfileSwitch.color())
LegendEntry(stringResource(R.string.timeline_legend_conn), TimelineEventKind.ConnectionEvent.color())
}
}
@@ -376,10 +377,10 @@ private fun StatusCheckRow(
private fun StatusPill(status: CheckStatus) {
val color = status.statusColor()
val label = when (status) {
CheckStatus.Pass -> "PASS"
CheckStatus.Warn -> "WARN"
CheckStatus.Fail -> "FAIL"
CheckStatus.Unknown -> "UNKNOWN"
CheckStatus.Pass -> stringResource(R.string.timeline_pass)
CheckStatus.Warn -> stringResource(R.string.timeline_warn)
CheckStatus.Fail -> stringResource(R.string.timeline_fail)
CheckStatus.Unknown -> stringResource(R.string.timeline_status_unknown)
}
Surface(
shape = RoundedCornerShape(50),
@@ -396,15 +397,16 @@ private fun StatusPill(status: CheckStatus) {
}
/** One-line "N failing · N warning · N passing" summary for the header. */
@Composable
private fun statusSummary(checks: List<StatusCheck>): String {
if (checks.isEmpty()) return "no checks"
if (checks.isEmpty()) return stringResource(R.string.timeline_no_checks)
val fail = checks.count { it.status == CheckStatus.Fail }
val warn = checks.count { it.status == CheckStatus.Warn }
val pass = checks.count { it.status == CheckStatus.Pass }
return buildList {
if (fail > 0) add("$fail failing")
if (warn > 0) add("$warn warning")
add("$pass passing")
if (fail > 0) add(pluralStringResource(R.plurals.timeline_summary_fail, fail, fail))
if (warn > 0) add(pluralStringResource(R.plurals.timeline_summary_warn, warn, warn))
add(pluralStringResource(R.plurals.timeline_summary_pass, pass, pass))
}.joinToString(" · ")
}
@@ -459,7 +461,7 @@ private fun TimelineRow(
modifier = Modifier.width(56.dp),
)
Text(
text = primary.title,
text = localizeTimelineTitle(primary.title),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurface,
maxLines = 1,
@@ -509,7 +511,7 @@ private fun TimelineRow(
.background(sibling.kind.color()),
)
Text(
text = sibling.title,
text = localizeTimelineTitle(sibling.title),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurface,
)
@@ -144,7 +144,7 @@ fun ToolProgressCard(
val riskDescription = toolCall.outputRisk?.let {
stringResource(R.string.tool_output_risk_a11y, it)
}.orEmpty()
val toolDescription = stringResource(R.string.tool_a11y, toolCall.name, statusText, durationDescription) +
val toolDescription = stringResource(R.string.tool_a11y, localizeToolName(toolCall.name), statusText, durationDescription) +
riskDescription
Card(
@@ -177,7 +177,7 @@ fun ToolProgressCard(
// Tool name — tool.generating may arrive nameless
Text(
text = if (isPreparing) toolCall.name.ifBlank { "Preparing tool…" } else toolCall.name,
text = if (isPreparing) localizeToolName(toolCall.name).ifBlank { stringResource(R.string.tool_preparing) } else localizeToolName(toolCall.name),
style = MaterialTheme.typography.labelMedium,
color = if (isPreparing) MaterialTheme.colorScheme.onSurfaceVariant
else androidx.compose.ui.graphics.Color.Unspecified,
@@ -211,7 +211,7 @@ fun ToolProgressCard(
// Expand/collapse
Icon(
imageVector = if (expanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore,
contentDescription = if (expanded) "Collapse" else "Expand",
contentDescription = if (expanded) stringResource(R.string.tool_progress_cd_collapse) else stringResource(R.string.tool_progress_cd_expand),
modifier = Modifier.size(16.dp),
tint = MaterialTheme.colorScheme.onSurfaceVariant
)
@@ -20,8 +20,6 @@ import androidx.compose.foundation.gestures.awaitFirstDown
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.ExperimentalLayoutApi
import androidx.compose.foundation.layout.FlowRow
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
@@ -37,16 +35,20 @@ import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.CenterFocusStrong
import androidx.compose.material.icons.filled.Close
import androidx.compose.material.icons.filled.Description
import androidx.compose.material.icons.filled.ExpandLess
import androidx.compose.material.icons.filled.ExpandMore
import androidx.compose.material.icons.filled.GraphicEq
import androidx.compose.material.icons.filled.Image
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Settings
import androidx.compose.material.icons.filled.Stop
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
@@ -72,11 +74,15 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.compose.ui.zIndex
import com.hermesandroid.relay.data.ChatMessage
import com.hermesandroid.relay.data.HermesCardAction
import com.hermesandroid.relay.data.MessageRole
import com.hermesandroid.relay.data.ToolCall
import com.hermesandroid.relay.data.VoicePresentationMode
import com.hermesandroid.relay.ui.components.avatar.AvatarRenderState
import com.hermesandroid.relay.ui.components.avatar.LocalAgentAvatar
import com.hermesandroid.relay.ui.components.avatar.LocalBackgroundVisualizationEnabled
import com.hermesandroid.relay.ui.LocalSnackbarHost
import com.hermesandroid.relay.ui.showHumanError
import com.hermesandroid.relay.util.HumanError
@@ -142,11 +148,11 @@ fun VoiceModeOverlay(
voiceOutputModel: String? = null,
voiceOutputVoice: String? = null,
voiceProfileName: String? = null,
voiceConfigScope: String? = null,
voiceOutputEnabled: Boolean? = null,
voiceOutputFallbackEnabled: Boolean? = null,
onOverlayRequest: () -> Unit = {},
onCompactModeChange: (Boolean) -> Unit = {},
presentationMode: VoicePresentationMode = VoicePresentationMode.Focus,
onPresentationModeChange: (VoicePresentationMode) -> Unit = {},
// === END PHASE3-voice-mode-transcript ===
// === v0.4.1 JIT permission-denied chip ===
// Tapped when the user clicks the permission-denied chip. Default no-op
@@ -159,22 +165,22 @@ fun VoiceModeOverlay(
onBackgroundRunCancel: () -> Unit = {},
onBackgroundRunTap: () -> Unit = {},
onHermesConfirmationAnswer: (String) -> Unit = {},
onCardAction: (messageId: String, cardKey: String, action: HermesCardAction) -> Unit =
{ _, _, _ -> },
onCardInput: (messageId: String, cardKey: String, value: String) -> Unit =
{ _, _, _ -> },
// === END v0.4.1 ===
) {
val backgroundVisualizationEnabled = LocalBackgroundVisualizationEnabled.current
val surface = MaterialTheme.colorScheme.surface
val haptic = LocalHapticFeedback.current
var controlsExpanded by remember { mutableStateOf(false) }
var focusMode by remember { mutableStateOf(true) }
val focusMode = presentationMode == VoicePresentationMode.Focus
val setFocusMode: (Boolean) -> Unit = { focused ->
focusMode = focused
onCompactModeChange(!focused)
}
LaunchedEffect(uiState.voiceMode) {
if (!uiState.voiceMode) {
setFocusMode(true)
}
onPresentationModeChange(
if (focused) VoicePresentationMode.Focus else VoicePresentationMode.Conversation,
)
}
// Voice errors surface ONLY on the overlay's own inline top banner
@@ -193,7 +199,7 @@ fun VoiceModeOverlay(
// chips, pill) didn't handle so stray taps/swipes don't fall
// through to the chat + session drawer behind it. Children run on
// the same Main pass leaf-first, so this only catches the gaps.
// In compact mode the overlay is intentionally transparent and the
// In Conversation the overlay is intentionally transparent and the
// chat stays interactive, so no scrim is installed.
.then(
if (focusMode) {
@@ -209,34 +215,36 @@ fun VoiceModeOverlay(
}
)
) {
VoiceSessionPill(
uiState = uiState,
expanded = controlsExpanded,
onExpandedChange = { controlsExpanded = it },
focusMode = focusMode,
onFocusModeChange = setFocusMode,
engineMode = voiceEngineMode,
provider = voiceOutputProvider,
model = voiceOutputModel,
voice = voiceOutputVoice,
profileName = voiceProfileName,
configScope = voiceConfigScope,
outputEnabled = voiceOutputEnabled,
fallbackEnabled = voiceOutputFallbackEnabled,
onModeChange = onModeChange,
onMicTap = onMicTap,
onMicRelease = onMicRelease,
onInterrupt = onInterrupt,
onPauseAutoMode = onPauseAutoMode,
onOverlayRequest = onOverlayRequest,
onOpenSettings = onOpenSettings,
onExit = onDismiss,
modifier = Modifier
.fillMaxWidth()
.statusBarsPadding()
.padding(horizontal = 12.dp, vertical = 8.dp)
.align(Alignment.TopCenter),
)
if (focusMode) {
VoiceSessionPill(
uiState = uiState,
expanded = controlsExpanded,
onExpandedChange = { controlsExpanded = it },
focusMode = true,
onFocusModeChange = setFocusMode,
engineMode = voiceEngineMode,
provider = voiceOutputProvider,
model = voiceOutputModel,
voice = voiceOutputVoice,
profileName = voiceProfileName,
outputEnabled = voiceOutputEnabled,
fallbackEnabled = voiceOutputFallbackEnabled,
onModeChange = onModeChange,
onMicTap = onMicTap,
onMicRelease = onMicRelease,
onInterrupt = onInterrupt,
onPauseAutoMode = onPauseAutoMode,
onOverlayRequest = onOverlayRequest,
onOpenSettings = onOpenSettings,
onExit = onDismiss,
modifier = Modifier
.fillMaxWidth()
.statusBarsPadding()
.padding(horizontal = 12.dp, vertical = 8.dp)
.zIndex(2f)
.align(Alignment.TopCenter),
)
}
// Center column: sphere -> waveform -> scrolling transcript.
//
@@ -317,14 +325,16 @@ fun VoiceModeOverlay(
.weight(1.0f),
contentAlignment = Alignment.Center,
) {
LocalAgentAvatar.current.Render(
state = AvatarRenderState(
state = voiceStateToSphereState(uiState.state),
voiceAmplitude = uiState.amplitude,
voiceMode = true,
),
modifier = Modifier.fillMaxSize(),
)
if (backgroundVisualizationEnabled) {
LocalAgentAvatar.current.Render(
state = AvatarRenderState(
state = voiceStateToSphereState(uiState.state),
voiceAmplitude = uiState.amplitude,
voiceMode = true,
),
modifier = Modifier.fillMaxSize(),
)
}
}
VoiceWaveform(
@@ -399,11 +409,21 @@ fun VoiceModeOverlay(
modifier = Modifier.fillMaxSize(),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
items(visibleTranscriptMessages, key = { it.id }) { msg ->
// Match ChatScreen's identity contract. A history
// reconcile can adopt the same authoritative server
// id into a live row while Compose still holds the
// pre-reconcile row for a frame. uiKey remains stable
// and unique across that transition.
items(visibleTranscriptMessages, key = ::voiceTranscriptItemKey) { msg ->
CompactTranscriptRow(
message = msg,
showThinking = showThinking,
expanded = msg.id == latestId || msg.isStreaming,
onViewConversation = {
onPresentationModeChange(VoicePresentationMode.Conversation)
},
onCardAction = onCardAction,
onCardInput = onCardInput,
)
}
if (pendingTranscriptText != null) {
@@ -418,6 +438,11 @@ fun VoiceModeOverlay(
),
showThinking = showThinking,
expanded = true,
onViewConversation = {
onPresentationModeChange(VoicePresentationMode.Conversation)
},
onCardAction = onCardAction,
onCardInput = onCardInput,
)
}
}
@@ -449,7 +474,7 @@ fun VoiceModeOverlay(
}
}
// Compact mode: the background-run chip must survive outside focus
// Conversation: the background-run chip must survive outside focus
// mode too — a running task with no visible presence reads as lost
// (the chip previously existed ONLY in the focus layout).
AnimatedVisibility(
@@ -458,7 +483,7 @@ fun VoiceModeOverlay(
exit = fadeOut(tween(180)),
modifier = Modifier
.align(Alignment.BottomCenter)
.padding(bottom = 120.dp, start = 24.dp, end = 24.dp),
.padding(bottom = 176.dp, start = 24.dp, end = 24.dp),
) {
BackgroundRunChip(
run = uiState.backgroundRun,
@@ -497,7 +522,7 @@ fun VoiceModeOverlay(
// retrying, so a failed/timed-out turn never traps the user
// on a retry-only banner.
TextButton(onClick = { onClearError() }) {
Text("Dismiss")
Text(stringResource(R.string.common_dismiss))
}
TextButton(
onClick = {
@@ -716,14 +741,247 @@ internal fun pendingVoiceTranscriptText(
return if (alreadyRendered) null else transcribed
}
internal fun voiceTranscriptItemKey(message: ChatMessage): String = message.uiKey
/**
* Conversation-mode voice controls that live inside the chat composer.
*
* The full voice surface keeps its top session pill. Conversation mode instead
* shares one bottom input system with text chat: a stateful mic is the primary
* action, while focus mode and detailed route/mode controls remain secondary.
* This keeps chat readable and removes the duplicate top-level mic affordance.
*/
@Composable
private fun InteractionMode.label(): String = when (this) {
InteractionMode.TapToTalk -> stringResource(R.string.voice_overlay_tap_to_talk)
InteractionMode.HoldToTalk -> stringResource(R.string.voice_overlay_hold_to_talk)
InteractionMode.Continuous -> stringResource(R.string.voice_overlay_continuous)
fun ConversationVoiceDock(
uiState: VoiceUiState,
engineMode: String?,
provider: String?,
model: String?,
voice: String?,
profileName: String?,
outputEnabled: Boolean?,
onMicTap: () -> Unit,
onMicRelease: () -> Unit,
onInterrupt: () -> Unit,
onPauseAutoMode: () -> Unit,
onModeChange: (InteractionMode) -> Unit,
onFocusRequest: () -> Unit,
onOverlayRequest: () -> Unit,
onOpenSettings: () -> Unit,
onExit: () -> Unit,
modifier: Modifier = Modifier,
) {
var expanded by remember { mutableStateOf(false) }
val engineText = voiceEngineLabel(engineMode)
val providerText = voiceProviderLabel(provider, model, voice, outputEnabled)
val profileText = profileName?.takeIf { it.isNotBlank() }
?: stringResource(R.string.voice_overlay_default_profile)
Column(
modifier = modifier
.fillMaxWidth()
.testTag("conversationVoiceDock"),
) {
AnimatedVisibility(visible = expanded) {
Column(
modifier = Modifier
.fillMaxWidth()
.padding(start = 10.dp, top = 8.dp, end = 10.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(6.dp),
) {
InteractionMode.values().forEach { mode ->
VoiceControlChip(
text = mode.shortLabel(),
selected = uiState.interactionMode == mode,
onClick = { onModeChange(mode) },
modifier = Modifier.weight(1f),
)
}
}
VoiceRouteSummary(
engine = engineText,
profile = profileText,
provider = providerText,
)
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
TextButton(
onClick = onOverlayRequest,
modifier = Modifier
.weight(1f)
.height(40.dp),
) {
Text(
text = stringResource(R.string.voice_overlay_overlay),
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
TextButton(
onClick = onExit,
modifier = Modifier
.weight(1f)
.height(40.dp),
) {
Text(
text = stringResource(R.string.voice_overlay_exit),
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
IconButton(
onClick = {
onExit()
onOpenSettings()
},
modifier = Modifier.size(40.dp),
) {
Icon(
imageVector = Icons.Filled.Settings,
contentDescription = stringResource(R.string.voice_overlay_settings_cd),
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
HorizontalDivider(
color = MaterialTheme.colorScheme.outlineVariant.copy(alpha = 0.72f),
)
}
}
AnimatedVisibility(visible = uiState.handoffStatus != null) {
VoiceHandoffStrip(
status = uiState.handoffStatus,
compact = true,
modifier = Modifier.padding(horizontal = 10.dp, vertical = 4.dp),
)
}
Row(
modifier = Modifier
.fillMaxWidth()
.height(58.dp)
.padding(horizontal = 8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Row(
modifier = Modifier.weight(1f),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(6.dp),
) {
val agentIconPath = LocalAgentIconPath.current
Surface(
modifier = Modifier.size(24.dp),
shape = CircleShape,
color = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.58f),
) {
if (!agentIconPath.isNullOrBlank()) {
AsyncImage(
model = File(agentIconPath),
contentDescription = null,
contentScale = ContentScale.Crop,
modifier = Modifier.clip(CircleShape),
)
} else {
Box(contentAlignment = Alignment.Center) {
Icon(
imageVector = Icons.Filled.GraphicEq,
contentDescription = null,
tint = MaterialTheme.colorScheme.primary,
modifier = Modifier.size(14.dp),
)
}
}
}
VoiceWaveform(
amplitude = uiState.amplitude,
state = uiState.state,
outputAudioActive = uiState.outputAudioActive,
height = 26.dp,
compactBars = true,
modifier = Modifier.width(40.dp),
)
Text(
text = conversationDockStateLabel(uiState.state),
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.tertiary,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
ConversationVoiceMicButton(
uiState = uiState,
onMicTap = onMicTap,
onMicRelease = onMicRelease,
onInterrupt = onInterrupt,
onPauseAutoMode = onPauseAutoMode,
baseSize = 52,
iconSize = 24,
modifier = Modifier.testTag("conversationVoiceDockMic"),
)
Row(
modifier = Modifier.weight(1f),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.End,
) {
TextButton(
onClick = onFocusRequest,
modifier = Modifier
.height(48.dp)
.testTag("conversationVoiceDockFocus"),
) {
Icon(
imageVector = Icons.Filled.CenterFocusStrong,
contentDescription = null,
modifier = Modifier.size(18.dp),
)
Spacer(Modifier.width(4.dp))
Text(
text = stringResource(R.string.voice_dock_focus),
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
IconButton(
onClick = { expanded = !expanded },
modifier = Modifier
.size(48.dp)
.testTag("conversationVoiceDockExpand"),
) {
Icon(
imageVector = if (expanded) Icons.Filled.ExpandMore else Icons.Filled.ExpandLess,
contentDescription = if (expanded) {
stringResource(R.string.voice_overlay_collapse_cd)
} else {
stringResource(R.string.voice_overlay_expand_cd)
},
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
}
}
}
@Composable
private fun conversationDockStateLabel(state: VoiceState): String = when (state) {
VoiceState.Idle -> stringResource(R.string.voice_overlay_state_ready)
VoiceState.Listening -> stringResource(R.string.voice_overlay_state_listening)
VoiceState.Transcribing -> stringResource(R.string.voice_overlay_state_transcribing)
VoiceState.Thinking -> stringResource(R.string.voice_overlay_state_thinking)
VoiceState.Speaking -> stringResource(R.string.voice_overlay_state_speaking)
VoiceState.Error -> stringResource(R.string.voice_overlay_state_error)
}
@OptIn(ExperimentalLayoutApi::class)
@Composable
private fun VoiceSessionPill(
uiState: VoiceUiState,
@@ -736,7 +994,6 @@ private fun VoiceSessionPill(
model: String?,
voice: String?,
profileName: String?,
configScope: String?,
outputEnabled: Boolean?,
fallbackEnabled: Boolean?,
onModeChange: (InteractionMode) -> Unit,
@@ -752,17 +1009,6 @@ private fun VoiceSessionPill(
val engineText = voiceEngineLabel(engineMode)
val providerText = voiceProviderLabel(provider, model, voice, outputEnabled)
val profileText = profileName?.takeIf { it.isNotBlank() } ?: stringResource(R.string.voice_overlay_default_profile)
val scopeText = when (configScope) {
"profile" -> stringResource(R.string.voice_overlay_profile_voice)
"relay" -> stringResource(R.string.voice_overlay_relay_voice)
"global" -> stringResource(R.string.voice_overlay_global_voice)
else -> null
}
val headlineText = if (focusMode) {
"$engineText / $profileText / $providerText"
} else {
"${stateHint(uiState.state).ifBlank { stringResource(R.string.voice_overlay_voice_ready) }} / $engineText / $providerText"
}
Surface(
modifier = modifier,
shape = RoundedCornerShape(24.dp),
@@ -804,23 +1050,30 @@ private fun VoiceSessionPill(
modifier = Modifier.size(18.dp),
)
}
Text(
text = stringResource(R.string.voice_overlay_voice),
style = MaterialTheme.typography.labelLarge,
color = MaterialTheme.colorScheme.onSurface,
)
// Collapsed header trimmed to icon + "Voice" + a single
// weighted title + chevron + close so the top bar can never
// wrap on narrow screens. The status pill and inline mic
// control moved into the expanded body below.
Text(
text = headlineText,
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.weight(1f),
)
Column(modifier = Modifier.weight(1f)) {
Text(
text = stringResource(R.string.voice_overlay_voice),
style = MaterialTheme.typography.labelLarge,
color = MaterialTheme.colorScheme.onSurface,
maxLines = 1,
)
Text(
text = "$engineText · $profileText",
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
if (!focusMode) {
ConversationVoiceMicButton(
uiState = uiState,
onMicTap = onMicTap,
onMicRelease = onMicRelease,
onInterrupt = onInterrupt,
onPauseAutoMode = onPauseAutoMode,
)
}
Icon(
imageVector = if (expanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore,
contentDescription = if (expanded) stringResource(R.string.voice_overlay_collapse_cd) else stringResource(R.string.voice_overlay_expand_cd),
@@ -855,28 +1108,6 @@ private fun VoiceSessionPill(
.padding(top = 10.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
// Moved out of the collapsed header (4a): the current
// interaction-mode pill plus the inline mic control. The
// compact mic only appears in compact mode, where the
// full-size bottom mic button is hidden.
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
StatusPill(uiState.interactionMode.label())
Spacer(Modifier.weight(1f))
if (!focusMode) {
CompactVoiceMicButton(
uiState = uiState,
onMicTap = onMicTap,
onMicRelease = onMicRelease,
onInterrupt = onInterrupt,
onPauseAutoMode = onPauseAutoMode,
)
}
}
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(6.dp),
@@ -891,31 +1122,34 @@ private fun VoiceSessionPill(
}
}
// Status pills use a FlowRow so they wrap to a second line
// on narrow screens instead of being squeezed into equal
// weighted columns that truncate every label (4a).
FlowRow(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(6.dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
) {
StatusPill(engineText, emphasized = true)
StatusPill(profileText)
scopeText?.let { StatusPill(it) }
StatusPill(providerText)
}
VoiceRouteSummary(
engine = engineText,
profile = profileText,
provider = providerText,
)
HorizontalDivider(
color = MaterialTheme.colorScheme.outlineVariant.copy(alpha = 0.72f),
)
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
horizontalArrangement = Arrangement.spacedBy(4.dp),
) {
TextButton(
onClick = { onFocusModeChange(!focusMode) },
modifier = Modifier.weight(1f),
modifier = Modifier
.weight(1.35f)
.height(40.dp),
) {
Text(
if (focusMode) stringResource(R.string.voice_overlay_compact) else stringResource(R.string.voice_overlay_focus),
if (focusMode) {
stringResource(R.string.voice_overlay_conversation)
} else {
stringResource(R.string.voice_overlay_focus)
},
style = MaterialTheme.typography.labelMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
@@ -925,15 +1159,29 @@ private fun VoiceSessionPill(
onFocusModeChange(false)
onOverlayRequest()
},
modifier = Modifier.weight(1f),
modifier = Modifier
.weight(0.95f)
.height(40.dp),
) {
Text(stringResource(R.string.voice_overlay_overlay), maxLines = 1, overflow = TextOverflow.Ellipsis)
Text(
stringResource(R.string.voice_overlay_overlay),
style = MaterialTheme.typography.labelMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
TextButton(
onClick = onExit,
modifier = Modifier.weight(1f),
modifier = Modifier
.weight(0.75f)
.height(40.dp),
) {
Text(stringResource(R.string.voice_overlay_exit), maxLines = 1, overflow = TextOverflow.Ellipsis)
Text(
stringResource(R.string.voice_overlay_exit),
style = MaterialTheme.typography.labelMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
// Settings link (4c): exit voice mode before navigating
// so the overlay isn't left floating over the Voice
@@ -943,6 +1191,7 @@ private fun VoiceSessionPill(
onExit()
onOpenSettings()
},
modifier = Modifier.size(40.dp),
) {
Icon(
imageVector = Icons.Filled.Settings,
@@ -958,12 +1207,15 @@ private fun VoiceSessionPill(
}
@Composable
private fun CompactVoiceMicButton(
private fun ConversationVoiceMicButton(
uiState: VoiceUiState,
onMicTap: () -> Unit,
onMicRelease: () -> Unit,
onInterrupt: () -> Unit,
onPauseAutoMode: () -> Unit,
baseSize: Int = 40,
iconSize: Int = 20,
modifier: Modifier = Modifier,
) {
VoiceMicButton(
uiState = uiState,
@@ -987,8 +1239,9 @@ private fun CompactVoiceMicButton(
)
},
onHoldRelease = onMicRelease,
baseSize = 40,
iconSize = 20,
baseSize = baseSize,
iconSize = iconSize,
modifier = modifier,
)
}
@@ -1074,37 +1327,49 @@ private fun VoiceControlChip(
}
@Composable
private fun StatusPill(
text: String,
private fun VoiceRouteSummary(
engine: String,
profile: String,
provider: String,
modifier: Modifier = Modifier,
emphasized: Boolean = false,
) {
val providerLine = provider.replace(" / ", " · ")
Surface(
modifier = modifier.height(24.dp),
shape = RoundedCornerShape(999.dp),
color = if (emphasized) {
// Opaque — translucent status bubbles over the floating overlay were
// hard to read against the sphere/chat behind them.
MaterialTheme.colorScheme.tertiaryContainer
} else {
MaterialTheme.colorScheme.surfaceVariant
},
contentColor = if (emphasized) {
MaterialTheme.colorScheme.onTertiaryContainer
} else {
MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = modifier.fillMaxWidth(),
shape = RoundedCornerShape(12.dp),
color = MaterialTheme.colorScheme.surfaceVariant,
contentColor = MaterialTheme.colorScheme.onSurfaceVariant,
) {
Box(
contentAlignment = Alignment.Center,
modifier = Modifier.padding(horizontal = 8.dp),
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
Text(
text = text,
style = MaterialTheme.typography.labelSmall,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
Icon(
imageVector = Icons.Filled.GraphicEq,
contentDescription = null,
tint = MaterialTheme.colorScheme.primary,
modifier = Modifier.size(18.dp),
)
Column(
modifier = Modifier.weight(1f),
verticalArrangement = Arrangement.spacedBy(3.dp),
) {
Text(
text = "$engine · $profile",
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.onSurface,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
Text(
text = providerLine,
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
}
}
}
@@ -1183,13 +1448,37 @@ private fun voiceProviderLabel(
val providerPart = provider?.takeIf { it.isNotBlank() } ?: stringResource(R.string.voice_overlay_provider_placeholder)
val modelPart = model?.takeIf { it.isNotBlank() }
val voicePart = voice?.takeIf { it.isNotBlank() }
return listOfNotNull(providerPart, modelPart, voicePart).joinToString(" / ")
return listOfNotNull(providerPart, modelPart, voicePart)
.let(::distinctVoiceRouteParts)
.map(::voiceRouteDisplayLabel)
.joinToString(" / ")
}
internal fun distinctVoiceRouteParts(parts: List<String>): List<String> =
parts.distinctBy { value ->
value.lowercase().filter(Char::isLetterOrDigit)
}
internal fun voiceRouteDisplayLabel(value: String): String =
value
.split('_', '-')
.filter(String::isNotBlank)
.joinToString(" ") { token ->
when (token.lowercase()) {
"xai" -> "xAI"
"openai" -> "OpenAI"
"tts" -> "TTS"
"api" -> "API"
else -> token.replaceFirstChar { first ->
if (first.isLowerCase()) first.titlecase() else first.toString()
}
}
}
@Composable
private fun voiceEngineLabel(engineMode: String?): String = when (engineMode) {
"realtime_agent" -> stringResource(R.string.voice_overlay_engine_realtime)
"hermes_voice_output" -> stringResource(R.string.voice_overlay_engine_hermes)
"hermes_voice_output" -> stringResource(R.string.voice_overlay_engine_standard)
null, "" -> stringResource(R.string.voice_overlay_engine_placeholder)
else -> engineMode
.replace('_', ' ')
@@ -1224,6 +1513,9 @@ private fun CompactTranscriptRow(
message: ChatMessage,
showThinking: Boolean,
expanded: Boolean,
onViewConversation: () -> Unit,
onCardAction: (messageId: String, cardKey: String, action: HermesCardAction) -> Unit,
onCardInput: (messageId: String, cardKey: String, value: String) -> Unit,
) {
if (message.role == MessageRole.SYSTEM) return
@@ -1248,6 +1540,13 @@ private fun CompactTranscriptRow(
message.role == MessageRole.USER -> MaterialTheme.colorScheme.primary
else -> MaterialTheme.colorScheme.secondary
}
val (markdownBody, inlineImages) = remember(message.content, message.role) {
if (message.role == MessageRole.ASSISTANT) {
extractChatInlineImages(message.content)
} else {
message.content to emptyList()
}
}
Column(
modifier = Modifier.fillMaxWidth(),
@@ -1285,7 +1584,7 @@ private fun CompactTranscriptRow(
}
when {
isVoiceActionBubble && hasText -> MarkdownContent(
content = message.content,
content = markdownBody,
textColor = MaterialTheme.colorScheme.onSurface,
)
message.role == MessageRole.USER && hasText -> Text(
@@ -1295,17 +1594,131 @@ private fun CompactTranscriptRow(
maxLines = if (expanded) Int.MAX_VALUE else 3,
overflow = TextOverflow.Ellipsis,
)
hasText -> Text(
text = message.content,
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurface,
maxLines = if (expanded) Int.MAX_VALUE else 6,
overflow = TextOverflow.Ellipsis,
hasText -> MarkdownContent(
content = markdownBody,
textColor = MaterialTheme.colorScheme.onSurface,
)
}
message.cards.forEachIndexed { index, card ->
if (card.actions.isNotEmpty() || card.input != null) {
Spacer(Modifier.height(6.dp))
val cardKey = card.id ?: "idx:$index"
HermesCardBubble(
card = card,
cardKey = cardKey,
dispatches = message.cardDispatches,
onActionTap = { key, action ->
onCardAction(message.id, key, action)
},
onInputSubmit = { key, value ->
onCardInput(message.id, key, value)
},
maxWidth = 360.dp,
)
}
}
if (
message.attachments.isNotEmpty() ||
message.cards.any { it.actions.isEmpty() && it.input == null } ||
inlineImages.isNotEmpty()
) {
Spacer(Modifier.height(6.dp))
VoiceRichResultAffordance(
message = message,
onViewConversation = onViewConversation,
)
}
}
}
@Composable
private fun VoiceRichResultAffordance(
message: ChatMessage,
onViewConversation: () -> Unit,
) {
val inlineImages = remember(message.content) {
extractChatInlineImages(message.content).second
}
val previewModel = remember(message.attachments, inlineImages) {
message.attachments.firstOrNull { it.isImage && !it.cachedUri.isNullOrBlank() }?.cachedUri
?: inlineImages.firstOrNull {
it.src.startsWith("https://") || it.src.startsWith("http://")
}?.src
}
val label = when {
message.attachments.size + inlineImages.size > 1 ->
stringResource(
R.string.voice_overlay_rich_results_count,
message.attachments.size + inlineImages.size,
)
voiceRichResultUsesImageLabel(message, inlineImages.size) ->
stringResource(R.string.voice_overlay_image_ready)
message.attachments.isNotEmpty() ->
stringResource(R.string.voice_overlay_attachment_ready)
else -> stringResource(R.string.voice_overlay_rich_result_ready)
}
Surface(
modifier = Modifier
.fillMaxWidth()
.clickable(onClick = onViewConversation),
shape = RoundedCornerShape(10.dp),
color = MaterialTheme.colorScheme.primaryContainer.copy(alpha = 0.48f),
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
if (previewModel != null) {
AsyncImage(
model = previewModel,
contentDescription = null,
contentScale = ContentScale.Crop,
modifier = Modifier
.size(40.dp)
.clip(RoundedCornerShape(8.dp)),
)
} else {
Icon(
imageVector = if (voiceRichResultUsesImageLabel(message, inlineImages.size)) {
Icons.Filled.Image
} else {
Icons.Filled.Description
},
contentDescription = null,
tint = MaterialTheme.colorScheme.primary,
modifier = Modifier.size(20.dp),
)
}
Text(
text = label,
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.onSurface,
modifier = Modifier.weight(1f),
)
Text(
text = stringResource(R.string.voice_overlay_view_conversation),
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.primary,
)
}
}
}
/**
* Image-specific copy is valid only when the single surfaced result is an
* actual image attachment or an inline image. Documents and unknown MIME types
* intentionally fall back to neutral attachment copy.
*/
internal fun voiceRichResultUsesImageLabel(
message: ChatMessage,
inlineImageCount: Int,
): Boolean {
if (message.attachments.size + inlineImageCount != 1) return false
return inlineImageCount == 1 || message.attachments.singleOrNull()?.isImage == true
}
@Composable
private fun VoiceToolStatusRow(toolCall: ToolCall) {
// Internal enum-style status drives both color and the localized label
@@ -1631,7 +2044,7 @@ private fun BackgroundRunChip(
imageVector = Icons.Filled.Close,
// The VM treats ✕ on a DONE chip as a local dismiss,
// never a cancel — label it accordingly for TalkBack.
contentDescription = if (done) "Dismiss" else "Cancel background task",
contentDescription = if (done) stringResource(R.string.common_dismiss) else stringResource(R.string.voice_overlay_cancel_task_cd),
tint = MaterialTheme.colorScheme.onSecondaryContainer,
modifier = Modifier.size(16.dp),
)
@@ -216,8 +216,11 @@ fun VoiceWaveform(
if (compactBars) {
val barCount = 10
val gap = 3.dp.toPx()
val barWidth = 2.5.dp.toPx()
// Keep the compact waveform inside its assigned composer lane.
// The previous 52 dp drawing width overflowed the 32 dp canvas and
// visually collided with the adjacent state label.
val gap = 2.dp.toPx()
val barWidth = 2.dp.toPx()
val totalWidth = barWidth * barCount + gap * (barCount - 1)
val startX = ((width - totalWidth) / 2f).coerceAtLeast(0f)
repeat(barCount) { index ->
@@ -15,6 +15,23 @@ import com.hermesandroid.relay.ui.components.SphereState
*/
enum class AvatarSource { BUILT_IN, USER }
/**
* Pet-only horizontal travel. This is deliberately separate from
* [SphereState]: `run`/`running` describes agent work, while these values
* describe the floating companion moving across the screen.
*/
enum class PetLocomotion {
None,
WalkLeft,
WalkRight,
RunLeft,
RunRight,
Jump,
Fall,
Held,
Wave,
}
/**
* Per-frame reactive input bundle handed to [AgentAvatar.Render].
*
@@ -28,6 +45,10 @@ enum class AvatarSource { BUILT_IN, USER }
* @property toolCallBurst tool-call pulse spike, slow decay.
* @property voiceAmplitude live mic/output amplitude (0..1) in voice mode.
* @property voiceMode whether a voice session is active (expands/animates the avatar).
* @property petLocomotion optional pet-only manipulation/travel pose. The
* floating host emits it under the explicit priority: user manipulation,
* agent activity, response visit, roam, then idle. Direct manipulation stays
* visible, while ambient travel never masks agent work.
* @property paused render a single still frame instead of animating — the
* avatar-agnostic reduced-motion signal. The sphere honors it by pinning its
* time/color phase; a sprite "pet" (C3) honors it by freezing its clip. This
@@ -41,21 +62,14 @@ data class AvatarRenderState(
val toolCallBurst: Float = 0f,
val voiceAmplitude: Float = 0f,
val voiceMode: Boolean = false,
val petLocomotion: PetLocomotion = PetLocomotion.None,
val paused: Boolean = false,
)
/**
* Swappable agent avatar — the visual embodiment of the agent across chat, the
* clean text-flow mode, and the voice overlay.
*
* This is the seam that lets the rendering implementation change without
* touching any surface: every surface composes `LocalAgentAvatar.current.Render(
* AvatarRenderState(...), modifier)` and is blind to whether the avatar is the
* ASCII [SphereAvatar] or a future sprite/Lottie "pet".
*
* Two-level model: **which avatar** (this interface) → for the sphere avatar,
* **which skin** (the unchanged [com.hermesandroid.relay.ui.components.SphereSkin]
* system, consumed internally by [SphereAvatar]).
* Shared reactive-rendering contract for the ambient Sphere and floating pets.
* Profile identity images use a separate UI path; implementing this interface
* does not make a visual the message sender's identity.
*
* Implementations are expected to be cheap, stable singletons (or `@Immutable`
* data) so they sit safely in a [staticCompositionLocalOf].
@@ -90,19 +104,30 @@ interface AgentAvatar {
}
/**
* The active agent avatar. Defaults to [SphereAvatar] so previews and any
* composable outside the app root render the classic orb; `RelayApp` provides
* the user's resolved choice beside the sphere-skin locals.
* Legacy ambient-visualization seam. The app now provides [SphereAvatar] here;
* floating companions are published through [LocalFloatingPet].
*/
val LocalAgentAvatar = staticCompositionLocalOf<AgentAvatar> { SphereAvatar }
/**
* Every selectable avatar (built-ins + any loaded user "pets"). Provided at the
* app root for the Appearance picker. C2 ships only [SphereAvatar]; C3 appends
* user pets.
* Compatibility list retained during the Sphere/pet split. New companion
* pickers should consume [LocalAvailablePets].
*/
val LocalAvailableAvatars = staticCompositionLocalOf<List<AgentAvatar>> { listOf(SphereAvatar) }
/**
* The optional floating pet companion. Unlike [LocalAgentAvatar], this does not
* replace the ambient sphere or the active profile's identity image. `null`
* means the user has not selected a companion.
*/
val LocalFloatingPet = staticCompositionLocalOf<AgentAvatar?> { null }
/** User-installed pets available to the companion picker (sphere excluded). */
val LocalAvailablePets = staticCompositionLocalOf<List<AgentAvatar>> { emptyList() }
/** Ambient sphere/background visibility, independent of the motion setting. */
val LocalBackgroundVisualizationEnabled = staticCompositionLocalOf { true }
/**
* Global pet playback-speed multiplier (1.0 = the clip's authored fps), set in
* Appearance and provided at the app root. [PetAvatar] scales its frame rate by
@@ -8,7 +8,6 @@ import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.produceState
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue
@@ -17,6 +16,7 @@ import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.graphics.drawscope.DrawScope
import androidx.compose.ui.graphics.drawscope.scale
import androidx.compose.ui.unit.IntOffset
import androidx.compose.ui.unit.IntSize
import com.hermesandroid.relay.ui.components.SphereReactivity
@@ -25,6 +25,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import java.io.File
import java.util.LinkedHashMap
import kotlin.math.roundToInt
/** Never animate a pet faster than this regardless of the spec's `fps`. */
@@ -58,9 +59,35 @@ private const val ONE_SHOT_MAX_MS = 4000L
/** How long to show a one-frame one-shot reaction before returning to the base loop. */
private const val ONE_SHOT_STILL_MS = 1800L
/** Runtime guardrails also protect sideloaded custom packs, not only Petdex downloads. */
internal const val PET_MAX_FRAME_SEQUENCE_FRAMES = 120
internal const val PET_MAX_FRAME_SEQUENCE_PIXELS = 8L * 1024L * 1024L
internal const val PET_MAX_SPRITE_SHEET_PIXELS = 16L * 1024L * 1024L
private const val PET_DECODED_CLIP_CACHE_ENTRIES = 4
private const val PET_DECODED_SHEET_CACHE_ENTRIES = 2
internal fun petOneShotReleaseDelayMs(frameCount: Int): Long =
if (frameCount <= 1) ONE_SHOT_STILL_MS else ONE_SHOT_MAX_MS
/** Reject invalid dimensions and cumulative decode requests before allocating a bitmap. */
internal fun petBitmapFitsPixelBudget(
width: Int,
height: Int,
usedPixels: Long,
maximumPixels: Long,
): Boolean {
if (width <= 0 || height <= 0 || usedPixels < 0L || maximumPixels <= 0L) return false
val pixels = width.toLong() * height.toLong()
return pixels > 0L && usedPixels <= maximumPixels && pixels <= maximumPixels - usedPixels
}
/** Keep the current visual until the requested clip is decoded; null clips intentionally clear it. */
internal fun <T> retainPetFrameDuringDecode(
previous: T?,
decoded: T?,
hasRequestedClip: Boolean,
): T? = if (hasRequestedClip) decoded ?: previous else null
/**
* The live reactive signals the pet renderer actually consumes **today**. A pet's
* effective [AgentAvatar.reactivity] is clamped to this in [PetSpec.toAvatar]
@@ -125,13 +152,21 @@ data class SpriteSheetClip(
val frameHeight: Int,
override val frameCount: Int,
override val fps: Float,
/** Zero-based first cell in the atlas, read row-major. */
val startFrame: Int = 0,
) : PetClip
internal data class PetClipSelection(
val clip: PetClip?,
val mirrorHorizontally: Boolean = false,
/** Manifest key that produced [clip], used by the capability preview. */
val sourceKey: String? = null,
)
/**
* User-loaded "pet" avatar — a bitmap frame-sequence / sprite-atlas companion
* that replaces the sphere. Implements the C2 [AgentAvatar] seam, so once
* selected it renders across chat, clean mode, the voice overlay, onboarding,
* and the splash with no per-surface code.
* User-loaded bitmap frame-sequence / sprite-atlas companion. It renders in the
* floating-pet surface and remains separate from profile identity and the
* optional ambient Sphere.
*
* Rendering is deliberately dependency-free: frames are decoded with
* [BitmapFactory] and drawn through a Compose [Canvas]. The frame loop is
@@ -141,7 +176,7 @@ data class SpriteSheetClip(
*
* Built by [PetSpec.toAvatar]; discovered by [PetLoader].
*
* @property clips a fully-resolved clip for every [SphereState] (the loader
* @property activityClips a fully-resolved clip for every [SphereState] (the loader
* pre-applies the idle/thinking/speaking fallback chain), so [Render] is a
* simple lookup.
* @property workingClip optional distinct "agent is running a tool" loop. When
@@ -153,21 +188,135 @@ data class SpriteSheetClip(
* the base loop on its trigger, then returns; one-frame clips show as a brief
* still reaction. Empty → no reactions (today's behavior). Triggers are
* derived from activity-state transitions in [Render].
* @property locomotionClips optional manipulation and directional travel clips.
* They take priority over agent activity, keeping direct movement visible and
* the upstream `running-left` / `running-right` rows separate from the
* in-place agent-work `running` row.
* @property legacyTravelClip optional `run`/`running` fallback for packs without
* directional rows. Tool-only `working` is deliberately excluded.
*/
class PetAvatar(
override val id: String,
override val label: String,
override val description: String,
override val reactivity: SphereReactivity,
private val clips: Map<SphereState, PetClip>,
private val workingClip: PetClip? = null,
private val oneShots: Map<PetOneShot, PetClip> = emptyMap(),
internal val activityClips: Map<SphereState, PetClip>,
internal val activityClipSources: Map<SphereState, String> = emptyMap(),
internal val workingClip: PetClip? = null,
internal val workingClipSource: String? = null,
internal val legacyTravelClip: PetClip? = null,
internal val legacyTravelClipSource: String? = null,
internal val locomotionClips: Map<PetLocomotion, PetClip> = emptyMap(),
internal val locomotionClipSources: Map<PetLocomotion, String> = emptyMap(),
internal val oneShots: Map<PetOneShot, PetClip> = emptyMap(),
) : AgentAvatar {
override val source: AvatarSource = AvatarSource.USER
private val decodedSheets = LinkedHashMap<String, DecodedSheet>(4, 0.75f, true)
private val decodedClips = LinkedHashMap<PetDecodeKey, PetFrames>(8, 0.75f, true)
private fun decode(clip: PetClip, stabilize: Boolean): PetFrames? {
val key = PetDecodeKey(clip, stabilize)
synchronized(decodedClips) { decodedClips[key] }?.let { return it }
val decoded = decodeClip(clip, stabilize, decodedSheets) ?: return null
synchronized(decodedClips) {
decodedClips[key] = decoded
while (decodedClips.size > PET_DECODED_CLIP_CACHE_ENTRIES) {
decodedClips.remove(decodedClips.keys.first())
}
}
return decoded
}
/** A one-shot is fireable only if it ships a clip that can actually show. */
private fun fireable(kind: PetOneShot): Boolean = (oneShots[kind]?.frameCount ?: 0) > 0
/** Resolve sustained motion without one-shot overlays; pure for focused tests. */
internal fun resolveBaseSelection(state: AvatarRenderState): PetClipSelection {
when (state.petLocomotion) {
PetLocomotion.Held -> {
return PetClipSelection(
locomotionClips[PetLocomotion.Held] ?: activityClips[SphereState.Idle],
sourceKey = locomotionClipSources[PetLocomotion.Held]
?: activityClipSources[SphereState.Idle],
)
}
PetLocomotion.Wave -> {
return PetClipSelection(
locomotionClips[PetLocomotion.Wave] ?: activityClips[SphereState.Idle],
sourceKey = locomotionClipSources[PetLocomotion.Wave]
?: activityClipSources[SphereState.Idle],
)
}
PetLocomotion.Jump -> {
return PetClipSelection(
locomotionClips[PetLocomotion.Jump] ?: activityClips[SphereState.Idle],
sourceKey = locomotionClipSources[PetLocomotion.Jump]
?: activityClipSources[SphereState.Idle],
)
}
PetLocomotion.Fall -> {
val clip = locomotionClips[PetLocomotion.Fall]
?: locomotionClips[PetLocomotion.Jump]
?: activityClips[SphereState.Idle]
val source = locomotionClipSources[PetLocomotion.Fall]
?: locomotionClipSources[PetLocomotion.Jump]
?: activityClipSources[SphereState.Idle]
return PetClipSelection(clip, sourceKey = source)
}
PetLocomotion.WalkLeft,
PetLocomotion.WalkRight,
PetLocomotion.RunLeft,
PetLocomotion.RunRight -> {
val exact = state.petLocomotion
val movingRight = exact == PetLocomotion.WalkRight || exact == PetLocomotion.RunRight
val sameDirectionAlternate = when (exact) {
PetLocomotion.WalkLeft -> PetLocomotion.RunLeft
PetLocomotion.WalkRight -> PetLocomotion.RunRight
PetLocomotion.RunLeft -> PetLocomotion.WalkLeft
PetLocomotion.RunRight -> PetLocomotion.WalkRight
}
val sameDirectionMotion = listOf(exact, sameDirectionAlternate)
.firstOrNull { locomotionClips[it] != null }
if (sameDirectionMotion != null) {
return PetClipSelection(
clip = locomotionClips[sameDirectionMotion],
sourceKey = locomotionClipSources[sameDirectionMotion],
)
}
val oppositeDirection = when (exact) {
PetLocomotion.WalkLeft -> listOf(PetLocomotion.WalkRight, PetLocomotion.RunRight)
PetLocomotion.WalkRight -> listOf(PetLocomotion.WalkLeft, PetLocomotion.RunLeft)
PetLocomotion.RunLeft -> listOf(PetLocomotion.RunRight, PetLocomotion.WalkRight)
PetLocomotion.RunRight -> listOf(PetLocomotion.RunLeft, PetLocomotion.WalkLeft)
}
val oppositeMotion = oppositeDirection.firstOrNull { locomotionClips[it] != null }
if (oppositeMotion != null) {
return PetClipSelection(
clip = locomotionClips[oppositeMotion],
mirrorHorizontally = true,
sourceKey = locomotionClipSources[oppositeMotion],
)
}
// Upstream treats the legacy in-place run row as left-facing
// for travel, mirroring it only for rightward motion.
return PetClipSelection(
clip = legacyTravelClip ?: activityClips[SphereState.Idle],
mirrorHorizontally = legacyTravelClip != null && movingRight,
sourceKey = legacyTravelClipSource ?: activityClipSources[SphereState.Idle],
)
}
PetLocomotion.None -> Unit
}
val toolActive = workingClip != null &&
state.toolCallBurst >= WORKING_BURST_THRESHOLD &&
(state.state == SphereState.Thinking || state.state == SphereState.Streaming)
val clip = if (toolActive) workingClip else (activityClips[state.state] ?: activityClips[SphereState.Idle])
val source = if (toolActive) workingClipSource else activityClipSources[state.state]
return PetClipSelection(clip, sourceKey = source)
}
internal fun resolveBaseClip(state: AvatarRenderState): PetClip? = resolveBaseSelection(state).clip
@Composable
override fun Render(state: AvatarRenderState, modifier: Modifier) {
// ── One-shot reactions ──────────────────────────────────────────────
@@ -202,33 +351,42 @@ class PetAvatar(
// `working` clip if the pet ships one; otherwise the base-state clip.
// toolCallBurst is ~0 outside tool activity, and Error keeps its own clip,
// so this only fires while the agent actually operates a tool.
val toolActive = workingClip != null &&
state.toolCallBurst >= WORKING_BURST_THRESHOLD &&
(state.state == SphereState.Thinking || state.state == SphereState.Streaming)
// A live reaction overlays everything; otherwise working overlays the base.
val oneShotClip = activeOneShot?.let { oneShots[it] }
val baseClip = if (toolActive) workingClip else (clips[state.state] ?: clips[SphereState.Idle])
val clip = oneShotClip ?: baseClip
val baseSelection = resolveBaseSelection(state)
val clip = oneShotClip ?: baseSelection.clip
val mirrorHorizontally = oneShotClip == null && baseSelection.mirrorHorizontally
val playOnce = oneShotClip != null
// Re-center each frame on its own opaque content at decode time —
// neutralizes the positional drift common in AI-generated sheets (a
// character that floats/jumps cell-to-cell). Global Appearance toggle;
// flipping it re-decodes via the produceState key.
// flipping it selects a separately cached decode.
val stabilize = LocalPetStabilize.current
// Decode the active clip off the main thread; null until ready / on
// decode failure (graceful — the avatar just renders nothing).
val frames by produceState<PetFrames?>(initialValue = null, clip, stabilize) {
value = if (clip == null) {
// Decode the active clip off the main thread. Keep the last complete
// visual while a new state is loading so greet/done/locomotion swaps do
// not expose the Canvas as a blank frame.
var displayedClip by remember(id, stabilize) {
mutableStateOf<DisplayedPetClip?>(null)
}
LaunchedEffect(id, clip, stabilize, mirrorHorizontally) {
val decoded = if (clip == null) {
null
} else {
withContext(Dispatchers.IO) { runCatching { decodeClip(clip, stabilize) }.getOrNull() }
withContext(Dispatchers.IO) { runCatching { decode(clip, stabilize) }.getOrNull() }
?.let { DisplayedPetClip(it, mirrorHorizontally) }
}
displayedClip = retainPetFrameDuringDecode(
previous = displayedClip,
decoded = decoded,
hasRequestedClip = clip != null,
)
}
var frameIndex by remember(clip) { mutableIntStateOf(0) }
val current = frames
val current = displayedClip?.frames
val displayedMirror = displayedClip?.mirrorHorizontally == true
var frameIndex by remember(current) { mutableIntStateOf(0) }
val animate = current != null && current.frameCount > 1 && !state.paused
// Live activity level + opt-in speedup, read inside the loop so playback
@@ -245,7 +403,7 @@ class PetAvatar(
// the base loop wraps with modulo. With intensity modulation on, fps
// is recomputed each tick from the live activity level.
LaunchedEffect(current, playOnce) {
val f = current ?: return@LaunchedEffect
val f = current
val baseFps = f.fps.coerceIn(1f, PET_MAX_FPS)
var acc = 0f
var last = withFrameNanos { it }
@@ -293,7 +451,14 @@ class PetAvatar(
Canvas(modifier = modifier) {
val f = current ?: return@Canvas
drawPetFrame(f, frameIndex.coerceIn(0, (f.frameCount - 1).coerceAtLeast(0)), bounce)
val index = frameIndex.coerceIn(0, (f.frameCount - 1).coerceAtLeast(0))
if (displayedMirror) {
scale(scaleX = -1f, scaleY = 1f, pivot = center) {
drawPetFrame(f, index, bounce)
}
} else {
drawPetFrame(f, index, bounce)
}
}
}
}
@@ -306,13 +471,55 @@ private class PetFrames(
val frameHeight: Int,
val frameCount: Int,
val fps: Float,
val startFrame: Int = 0,
/** Per-frame recenter offset (source px) when stabilization is on; null = off. */
val centerOffsets: List<IntOffset>? = null,
)
private fun decodeClip(clip: PetClip, stabilize: Boolean): PetFrames? = when (clip) {
private data class DisplayedPetClip(
val frames: PetFrames,
val mirrorHorizontally: Boolean,
)
private data class PetDecodeKey(
val clip: PetClip,
val stabilize: Boolean,
)
/** One decoded atlas per selected pet; all of its row clips share these pixels. */
private data class DecodedSheet(val bitmap: Bitmap, val image: ImageBitmap)
private fun decodeClip(
clip: PetClip,
stabilize: Boolean,
decodedSheets: MutableMap<String, DecodedSheet>,
): PetFrames? = when (clip) {
is FrameSequenceClip -> {
val bitmaps = clip.files.mapNotNull { file -> BitmapFactory.decodeFile(file.absolutePath) }
if (clip.files.isEmpty() || clip.files.size > PET_MAX_FRAME_SEQUENCE_FRAMES) return null
val bitmaps = mutableListOf<Bitmap>()
var usedPixels = 0L
for (file in clip.files) {
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeFile(file.absolutePath, bounds)
if (
!petBitmapFitsPixelBudget(
width = bounds.outWidth,
height = bounds.outHeight,
usedPixels = usedPixels,
maximumPixels = PET_MAX_FRAME_SEQUENCE_PIXELS,
)
) {
bitmaps.forEach(Bitmap::recycle)
return null
}
val bitmap = BitmapFactory.decodeFile(file.absolutePath)
if (bitmap == null) {
bitmaps.forEach(Bitmap::recycle)
return null
}
bitmaps += bitmap
usedPixels += bounds.outWidth.toLong() * bounds.outHeight.toLong()
}
if (bitmaps.isEmpty()) {
null
} else {
@@ -329,23 +536,55 @@ private fun decodeClip(clip: PetClip, stabilize: Boolean): PetFrames? = when (cl
}
is SpriteSheetClip -> {
val bmp = BitmapFactory.decodeFile(clip.sheet.absolutePath)
if (bmp == null) {
if (
clip.startFrame < 0 || clip.frameWidth <= 0 || clip.frameHeight <= 0 ||
clip.frameCount <= 0
) return null
val decoded = synchronized(decodedSheets) {
decodedSheets[clip.sheet.absolutePath] ?: run {
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeFile(clip.sheet.absolutePath, bounds)
if (
!petBitmapFitsPixelBudget(
width = bounds.outWidth,
height = bounds.outHeight,
usedPixels = 0L,
maximumPixels = PET_MAX_SPRITE_SHEET_PIXELS,
)
) {
null
} else {
BitmapFactory.decodeFile(clip.sheet.absolutePath)?.let { bitmap ->
DecodedSheet(bitmap, bitmap.asImageBitmap()).also {
decodedSheets[clip.sheet.absolutePath] = it
while (decodedSheets.size > PET_DECODED_SHEET_CACHE_ENTRIES) {
decodedSheets.remove(decodedSheets.keys.first())
}
}
}
}
}
}
if (decoded == null) {
null
} else {
val bmp = decoded.bitmap
// Clamp the declared frame count to what the sheet can actually hold.
val cols = (bmp.width / clip.frameWidth).coerceAtLeast(1)
val rows = (bmp.height / clip.frameHeight).coerceAtLeast(1)
val count = clip.frameCount.coerceIn(1, cols * rows)
val available = (cols * rows - clip.startFrame).coerceAtLeast(0)
if (available == 0) return null
val count = clip.frameCount.coerceIn(1, available)
PetFrames(
frames = emptyList(),
sheet = bmp.asImageBitmap(),
sheet = decoded.image,
frameWidth = clip.frameWidth,
frameHeight = clip.frameHeight,
frameCount = count,
fps = clip.fps,
startFrame = clip.startFrame,
centerOffsets = if (stabilize) {
sheetRecenter(bmp, cols, clip.frameWidth, clip.frameHeight, count)
sheetRecenter(bmp, cols, clip.frameWidth, clip.frameHeight, clip.startFrame, count)
} else {
null
},
@@ -355,10 +594,18 @@ private fun decodeClip(clip: PetClip, stabilize: Boolean): PetFrames? = when (cl
}
/** Per-cell recenter offsets for a sprite sheet (see [contentRecenter]). */
private fun sheetRecenter(bmp: Bitmap, cols: Int, fw: Int, fh: Int, count: Int): List<IntOffset> {
private fun sheetRecenter(
bmp: Bitmap,
cols: Int,
fw: Int,
fh: Int,
startFrame: Int,
count: Int,
): List<IntOffset> {
val buf = IntArray(fw * fh)
return (0 until count).map { i ->
contentRecenter(bmp, (i % cols) * fw, (i / cols) * fh, fw, fh, buf)
val cell = startFrame + i
contentRecenter(bmp, (cell % cols) * fw, (cell / cols) * fh, fw, fh, buf)
}
}
@@ -401,8 +648,9 @@ private fun DrawScope.drawPetFrame(f: PetFrames, index: Int, bounce: Float) {
val fh = f.frameHeight
if (fw <= 0 || fh <= 0) return
val cols = (f.sheet.width / fw).coerceAtLeast(1)
val col = index % cols
val row = index / cols
val cell = f.startFrame + index
val col = cell % cols
val row = cell / cols
val (dstOffset, dstSize) = containFit(fw.toFloat(), fh.toFloat(), bounce)
drawImage(
image = f.sheet,
@@ -0,0 +1,161 @@
package com.hermesandroid.relay.ui.components.avatar
import com.hermesandroid.relay.ui.components.SphereState
/** Stable preview actions backed by the same renderer inputs used by the overlay. */
enum class PetPreviewAction {
Idle,
WalkLeft,
WalkRight,
Jump,
Fall,
Held,
Wave,
Working,
Review,
Waiting,
Error,
}
/** How honestly the installed pack can show a requested preview action. */
enum class PetPreviewSupport {
Direct,
Mirrored,
Fallback,
MirroredFallback,
}
/**
* One interactive preview choice. [renderState] goes straight into [PetAvatar.Render],
* while [sourceKey] and [support] describe the exact clip selection it will make.
*/
data class PetPreviewMapping(
val action: PetPreviewAction,
val renderState: AvatarRenderState,
val sourceKey: String,
val support: PetPreviewSupport,
)
private data class PetPreviewDefinition(
val action: PetPreviewAction,
val renderState: AvatarRenderState,
val directSourceKeys: Set<String>,
)
private val PET_PREVIEW_DEFINITIONS = listOf(
PetPreviewDefinition(
PetPreviewAction.Idle,
AvatarRenderState(SphereState.Idle),
setOf("idle"),
),
PetPreviewDefinition(
PetPreviewAction.WalkLeft,
AvatarRenderState(
SphereState.Idle,
petLocomotion = PetLocomotion.WalkLeft,
),
setOf("walking-left", "walk-left", "running-left", "run-left"),
),
PetPreviewDefinition(
PetPreviewAction.WalkRight,
AvatarRenderState(
SphereState.Idle,
petLocomotion = PetLocomotion.WalkRight,
),
setOf("walking-right", "walk-right", "running-right", "run-right"),
),
PetPreviewDefinition(
PetPreviewAction.Jump,
AvatarRenderState(SphereState.Idle, petLocomotion = PetLocomotion.Jump),
setOf("jumping", "jump"),
),
PetPreviewDefinition(
PetPreviewAction.Fall,
AvatarRenderState(SphereState.Idle, petLocomotion = PetLocomotion.Fall),
setOf("falling", "fall"),
),
PetPreviewDefinition(
PetPreviewAction.Held,
AvatarRenderState(SphereState.Idle, petLocomotion = PetLocomotion.Held),
setOf("held", "hold"),
),
PetPreviewDefinition(
PetPreviewAction.Wave,
AvatarRenderState(SphereState.Idle, petLocomotion = PetLocomotion.Wave),
setOf("waving", "wave"),
),
PetPreviewDefinition(
PetPreviewAction.Working,
AvatarRenderState(SphereState.Streaming, toolCallBurst = 1f),
setOf("working", "run", "running"),
),
PetPreviewDefinition(
PetPreviewAction.Review,
AvatarRenderState(SphereState.Thinking),
setOf("thinking", "review"),
),
PetPreviewDefinition(
PetPreviewAction.Waiting,
AvatarRenderState(SphereState.Listening),
setOf("listening", "waiting"),
),
PetPreviewDefinition(
PetPreviewAction.Error,
AvatarRenderState(SphereState.Error),
setOf("error", "failed"),
),
)
private val PET_NATIVE_DIRECTIONAL_TRAVEL_KEYS = setOf(
"walking-left",
"walk-left",
"running-left",
"run-left",
"walking-right",
"walk-right",
"running-right",
"run-right",
)
/**
* Resolve every preview through [PetAvatar.resolveBaseSelection]. Keeping this as
* a pure projection means the browser cannot drift from overlay clip priority,
* directional mirroring, or fallback behavior.
*/
fun PetAvatar.previewMappings(): List<PetPreviewMapping> = PET_PREVIEW_DEFINITIONS.map { definition ->
val selection = resolveBaseSelection(definition.renderState)
val sourceKey = selection.sourceKey ?: "idle"
val sourceIsDirect = sourceKey in definition.directSourceKeys
val sourceIsNativeDirectionalTravel =
definition.action in setOf(PetPreviewAction.WalkLeft, PetPreviewAction.WalkRight) &&
sourceKey in PET_NATIVE_DIRECTIONAL_TRAVEL_KEYS
val support = when {
selection.mirrorHorizontally && sourceIsNativeDirectionalTravel -> PetPreviewSupport.Mirrored
selection.mirrorHorizontally -> PetPreviewSupport.MirroredFallback
sourceIsDirect -> PetPreviewSupport.Direct
else -> PetPreviewSupport.Fallback
}
PetPreviewMapping(
action = definition.action,
renderState = definition.renderState,
sourceKey = sourceKey,
support = support,
)
}
/** Renderer twin for manual state inspection; transient greet/done clips are intentionally absent. */
fun PetAvatar.forCapabilityPreview(): PetAvatar = PetAvatar(
id = id,
label = label,
description = description,
reactivity = reactivity,
activityClips = activityClips,
activityClipSources = activityClipSources,
workingClip = workingClip,
workingClipSource = workingClipSource,
legacyTravelClip = legacyTravelClip,
legacyTravelClipSource = legacyTravelClipSource,
locomotionClips = locomotionClips,
locomotionClipSources = locomotionClipSources,
oneShots = emptyMap(),
)
@@ -2,12 +2,15 @@ package com.hermesandroid.relay.ui.components.avatar
import android.content.Context
import android.util.Log
import com.hermesandroid.relay.petdex.PETDEX_CURRENT_FRAME_COUNTS
import com.hermesandroid.relay.petdex.PETDEX_CURRENT_LOOP_DURATIONS_MS
import com.hermesandroid.relay.ui.components.SphereReactivity
import com.hermesandroid.relay.ui.components.SphereState
import com.hermesandroid.relay.ui.components.UserContentDir
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import java.io.File
import kotlin.math.abs
/** The pet schema versions this build understands. */
const val PET_SPEC_SCHEMA_VERSION = 1
@@ -18,9 +21,10 @@ const val PET_SPEC_SCHEMA_VERSION = 1
* validated, and converted to a [PetAvatar] by [toAvatar]; see
* `docs/pet-spec.md` for the authoring reference.
*
* Clips are keyed by name in [states]; the loader maps the agent's six
* [SphereState]s onto the three core clips (`idle`/`thinking`/`speaking`) with a
* fallback chain, so a minimal pack only needs an `idle` clip.
* Clips are keyed by name in [states]; the loader accepts both the original
* Android names and Hermes/Petdex's current atlas taxonomy. Agent activity
* (`review`, `running`) and horizontal locomotion (`running-left` /
* `running-right`) remain separate. A minimal pack still needs only `idle`.
*
* Example:
* ```json
@@ -44,8 +48,12 @@ data class PetSpec(
val id: String = "",
val label: String = "",
val description: String = "",
/** Optional origin metadata for gallery-installed packs. Never fetched or executed. */
val source: String = "",
val sourceUrl: String = "",
val creator: String = "",
val reactive: PetReactiveSpec = PetReactiveSpec(),
/** Clip definitions keyed by `idle`/`thinking`/`speaking` (or a full state name). */
/** Clip definitions keyed by an Android activity name or Hermes/Petdex row name. */
val states: Map<String, PetClipSpec> = emptyMap(),
/** Fallback clip for any state with no usable clip in [states]. */
val defaults: PetClipSpec? = null,
@@ -75,21 +83,41 @@ data class PetClipSpec(
val frameWidth: Int = 0,
val frameHeight: Int = 0,
val frameCount: Int = 0,
/** Zero-based cell offset into [sheet], read row-major. */
val startFrame: Int = 0,
val fps: Float = 8f,
)
// Each agent state maps onto an ordered clip-name fallback chain ending at
// "idle" — so authors can supply just idle/thinking/speaking, or override any
// individual state by name. The Streaming state accepts a friendly "writing"
// alias (the agent producing output text) ahead of the internal "streaming"
// key. See docs/pet-spec.md "Agent states & pet behavior".
// Android activity names and upstream Hermes/Petdex row names are both accepted.
// `running` is the in-place agent-work row; directional rows are resolved by
// LOCOMOTION_CLIP_CHAIN and never substituted for agent activity.
private val STATE_CLIP_CHAIN: Map<SphereState, List<String>> = mapOf(
SphereState.Idle to listOf("idle"),
SphereState.Thinking to listOf("thinking", "idle"),
SphereState.Streaming to listOf("writing", "streaming", "speaking", "thinking", "idle"),
SphereState.Listening to listOf("listening", "idle"),
SphereState.Speaking to listOf("speaking", "writing", "thinking", "idle"),
SphereState.Error to listOf("error", "thinking", "idle"),
SphereState.Thinking to listOf("thinking", "review", "idle"),
SphereState.Streaming to listOf(
"writing",
"streaming",
"working",
"run",
"running",
"review",
"thinking",
"idle",
),
SphereState.Listening to listOf("listening", "waiting", "idle"),
SphereState.Speaking to listOf("speaking", "talking", "wave", "waving", "writing", "idle"),
SphereState.Error to listOf("error", "failed", "review", "thinking", "idle"),
)
private val LOCOMOTION_CLIP_CHAIN: Map<PetLocomotion, List<String>> = mapOf(
PetLocomotion.WalkLeft to listOf("walking-left", "walk-left", "running-left", "run-left"),
PetLocomotion.WalkRight to listOf("walking-right", "walk-right", "running-right", "run-right"),
PetLocomotion.RunLeft to listOf("running-left", "run-left", "walking-left", "walk-left"),
PetLocomotion.RunRight to listOf("running-right", "run-right", "walking-right", "walk-right"),
PetLocomotion.Jump to listOf("jumping", "jump"),
PetLocomotion.Fall to listOf("falling", "fall", "jumping", "jump"),
PetLocomotion.Held to listOf("held", "hold"),
PetLocomotion.Wave to listOf("waving", "wave"),
)
/**
@@ -104,26 +132,41 @@ fun PetSpec.toAvatar(dir: File): PetAvatar {
}
require(id.isNotBlank()) { "missing id" }
val resolvedLabel = label.ifBlank { id }
val runtimeSpec = normalizedPetdexAdapterManifest()
val idleClip = resolveStateClip(SphereState.Idle, dir)
requireNotNull(idleClip) {
val idleResolved = runtimeSpec.resolveStateClip(SphereState.Idle, dir)
requireNotNull(idleResolved) {
"no usable 'idle' clip — need a frames list or a sprite sheet whose files exist in the pack"
}
val clips = SphereState.entries.associateWith { state ->
resolveStateClip(state, dir) ?: idleClip
val resolvedActivities = SphereState.entries.associateWith { state ->
runtimeSpec.resolveStateClip(state, dir) ?: idleResolved
}
val clips = resolvedActivities.mapValues { it.value.clip }
val activitySources = resolvedActivities.mapValues { it.value.key }
// Opt-in tool-use clip: resolved only from an explicit `working` key (no
// fallback — without it there's no distinct tool behavior, and the pet just
// keeps its base-state clip during tool use, exactly as before).
val workingClip = states["working"]?.toClip(dir)
// The upstream in-place `run`/`running` row means agent work. It must not be
// confused with the dedicated left/right rows used for screen locomotion.
val workingResolved = runtimeSpec.resolveAliasClip(listOf("working", "run", "running"), dir)
// Only the canonical/legacy in-place run row may substitute for directional
// travel. A tool-specific `working` pose is never locomotion.
val legacyTravelResolved = runtimeSpec.resolveAliasClip(listOf("run", "running"), dir)
// Opt-in one-shot reaction clips, by friendly alias — resolved from explicit
// keys only (no fallback). Absent reactions simply don't play.
val resolvedLocomotion = LOCOMOTION_CLIP_CHAIN.mapNotNull { (motion, keys) ->
runtimeSpec.resolveAliasClip(keys, dir)?.let { motion to it }
}.toMap()
val locomotionClips = resolvedLocomotion.mapValues { it.value.clip }
val locomotionSources = resolvedLocomotion.mapValues { it.value.key }
// Hermes maps a clean completion/greeting to wave; jump is the stronger
// success/celebration fallback. Explicit Android names retain priority.
val oneShots = buildMap {
resolveAliasClip(listOf("greet", "wake"), dir)?.let { put(PetOneShot.Greet, it) }
resolveAliasClip(listOf("done", "celebrate"), dir)?.let { put(PetOneShot.Done, it) }
runtimeSpec.resolveAliasClip(listOf("greet", "wake", "wave", "waving"), dir)
?.let { put(PetOneShot.Greet, it.clip) }
runtimeSpec.resolveAliasClip(
listOf("done", "wave", "waving", "success", "celebrate", "jump", "jumping"),
dir,
)?.let { put(PetOneShot.Done, it.clip) }
}
return PetAvatar(
@@ -137,28 +180,69 @@ fun PetSpec.toAvatar(dir: File): PetAvatar {
// it ships one, so the badge is driven by the clip, not the declared flag.
reactivity = SphereReactivity(
voice = reactive.voice && PET_RENDERER_CAPABILITIES.voice,
tools = (workingClip != null) && PET_RENDERER_CAPABILITIES.tools,
tools = (workingResolved != null) && PET_RENDERER_CAPABILITIES.tools,
intensity = reactive.intensity && PET_RENDERER_CAPABILITIES.intensity,
gaze = false,
),
clips = clips,
workingClip = workingClip,
activityClips = clips,
activityClipSources = activitySources,
workingClip = workingResolved?.clip,
workingClipSource = workingResolved?.key,
legacyTravelClip = legacyTravelResolved?.clip,
legacyTravelClipSource = legacyTravelResolved?.key,
locomotionClips = locomotionClips,
locomotionClipSources = locomotionSources,
oneShots = oneShots,
)
}
private data class ResolvedPetClip(val clip: PetClip, val key: String)
/** First usable clip among [keys] (friendly aliases), or null. No fallback. */
private fun PetSpec.resolveAliasClip(keys: List<String>, dir: File): PetClip? {
for (key in keys) states[key]?.toClip(dir)?.let { return it }
private fun PetSpec.resolveAliasClip(keys: List<String>, dir: File): ResolvedPetClip? {
for (key in keys) states[key]?.toClip(dir)?.let { return ResolvedPetClip(it, key) }
return null
}
private fun PetSpec.resolveStateClip(state: SphereState, dir: File): PetClip? {
private fun PetSpec.resolveStateClip(state: SphereState, dir: File): ResolvedPetClip? {
for (key in STATE_CLIP_CHAIN[state] ?: listOf("idle")) {
val clip = states[key]?.toClip(dir)
if (clip != null) return clip
if (clip != null) return ResolvedPetClip(clip, key)
}
return defaults?.toClip(dir)
return defaults?.toClip(dir)?.let { ResolvedPetClip(it, "default") }
}
/**
* Repair manifests written by the first Android Petdex adapter, which declared
* six cells for every current row. Installed packs live outside the APK and
* survive upgrades, so correcting only new installs would leave wave/done on
* transparent cells and truncate the eight-frame travel/error rows.
*/
private fun PetSpec.normalizedPetdexAdapterManifest(): PetSpec {
if (source != "petdex") return this
val orderedKeys = PETDEX_CURRENT_FRAME_COUNTS.keys.toList()
val staleAdapterFps = 6f * 1000f / 1100f
val clips = orderedKeys.mapIndexed { row, key ->
val clip = states[key] ?: return this
if (
clip.sheet.isBlank() || clip.frameWidth != 192 || clip.frameHeight != 208 ||
clip.frameCount != 6 || clip.startFrame != row * 8 ||
abs(clip.fps - staleAdapterFps) > 0.01f
) return this
clip
}
if (clips.map { it.sheet }.distinct().size != 1) return this
return copy(
states = states.mapValues { (key, clip) ->
val canonicalCount = PETDEX_CURRENT_FRAME_COUNTS[key] ?: return@mapValues clip
val durationMs = PETDEX_CURRENT_LOOP_DURATIONS_MS.getValue(key)
clip.copy(
frameCount = canonicalCount,
fps = canonicalCount * 1000f / durationMs,
)
},
)
}
private fun PetClipSpec.toClip(dir: File): PetClip? {
@@ -168,10 +252,10 @@ private fun PetClipSpec.toClip(dir: File): PetClip? {
if (files.isEmpty()) return null
return FrameSequenceClip(files, fpsSafe)
}
if (sheet.isNotBlank() && frameWidth > 0 && frameHeight > 0 && frameCount > 0) {
if (sheet.isNotBlank() && frameWidth > 0 && frameHeight > 0 && frameCount > 0 && startFrame >= 0) {
val sheetFile = safeChild(dir, sheet) ?: return null
if (!sheetFile.isFile) return null
return SpriteSheetClip(sheetFile, frameWidth, frameHeight, frameCount, fpsSafe)
return SpriteSheetClip(sheetFile, frameWidth, frameHeight, frameCount, fpsSafe, startFrame)
}
return null
}
@@ -202,6 +286,7 @@ private fun safeChild(dir: File, name: String): File? {
object PetLoader {
private const val TAG = "PetLoader"
private const val DIR = "pets"
private const val PETDEX_TRANSACTION_PREFIX = ".petdex-"
private val json = Json {
ignoreUnknownKeys = true
@@ -221,7 +306,9 @@ object PetLoader {
* discovery/skip-invalid behavior is unit-testable against a temp directory.
*/
fun loadPets(dir: File): List<PetAvatar> {
val packs = dir.listFiles { file -> file.isDirectory } ?: return emptyList()
val packs = dir.listFiles { file ->
file.isDirectory && !file.name.startsWith(PETDEX_TRANSACTION_PREFIX)
} ?: return emptyList()
return packs.sortedBy { it.name }.mapNotNull { packDir ->
val manifest = File(packDir, "pet.json")
if (!manifest.isFile) return@mapNotNull null
@@ -246,7 +333,9 @@ object PetLoader {
/** Pure overload (no Android Context) for tests against a temp directory. */
fun deletePet(dir: File, avatarId: String): Boolean {
val packs = dir.listFiles { file -> file.isDirectory } ?: return false
val packs = dir.listFiles { file ->
file.isDirectory && !file.name.startsWith(PETDEX_TRANSACTION_PREFIX)
} ?: return false
for (packDir in packs) {
val manifest = File(packDir, "pet.json")
if (!manifest.isFile) continue
@@ -6,17 +6,16 @@ import com.hermesandroid.relay.ui.components.MorphingSphere
import com.hermesandroid.relay.ui.components.SphereReactivity
/**
* Default agent avatar — the ASCII [MorphingSphere].
* Default ambient visualization — the ASCII [MorphingSphere].
*
* Its [Render] is the existing sphere call verbatim: it forwards the reactive
* bundle and lets [MorphingSphere] pull colors/params from
* [com.hermesandroid.relay.ui.components.LocalSphereSkin] internally (the
* `skin` parameter defaults to `LocalSphereSkin.current`). So the entire skin
* system is untouched and simply nested one level below the avatar choice —
* pick the sphere avatar, then pick its skin.
* system remains independently selectable in Appearance.
*
* A singleton because it holds no per-instance state; it sits in
* [LocalAgentAvatar] and [LocalAvailableAvatars].
* [LocalAgentAvatar]. Floating pets use their own companion seam.
*/
object SphereAvatar : AgentAvatar {
override val id: String = "sphere"
@@ -0,0 +1,277 @@
package com.hermesandroid.relay.ui.components.pet
import androidx.compose.runtime.Stable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateMapOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import androidx.compose.runtime.staticCompositionLocalOf
import androidx.compose.ui.Modifier
import androidx.compose.ui.composed
import androidx.compose.ui.geometry.Rect
import androidx.compose.ui.layout.boundsInRoot
import androidx.compose.ui.layout.onGloballyPositioned
import androidx.compose.runtime.DisposableEffect
import com.hermesandroid.relay.ui.components.SphereState
import com.hermesandroid.relay.ui.components.avatar.AvatarRenderState
/** Live agent/render state published by the currently visible app surface. */
data class PetCompanionActivity(
val renderState: AvatarRenderState = AvatarRenderState(SphereState.Idle),
val scrolling: Boolean = false,
val hidden: Boolean = false,
)
internal const val PET_VISIT_COOLDOWN_MIN_MS = 12_000L
internal const val PET_VISIT_COOLDOWN_RANGE_MS = 8_001L
internal const val PET_VISIT_EXPIRY_WINDOW_MS = 20_000L
enum class PetVisitReadiness { CoolingDown, Ready, Expired }
/** One post-response visit request, timed only with monotonic elapsed time. */
data class PetVisitRequest(
val assistantUiKey: String,
val targetKey: String,
val notBeforeElapsedMs: Long,
val expiresAtElapsedMs: Long,
) {
fun readinessAt(nowElapsedMs: Long): PetVisitReadiness = when {
nowElapsedMs < notBeforeElapsedMs -> PetVisitReadiness.CoolingDown
nowElapsedMs <= expiresAtElapsedMs -> PetVisitReadiness.Ready
else -> PetVisitReadiness.Expired
}
}
/** Pure falling-edge detector and latest-wins request state. */
data class PetVisitRequestState(
val streamArmed: Boolean = false,
val lastRequestedAssistantUiKey: String? = null,
val pending: PetVisitRequest? = null,
)
internal fun deterministicPetVisitCooldownMs(assistantUiKey: String): Long {
val stableUnsignedHash = assistantUiKey.hashCode().toLong() and 0xffff_ffffL
return PET_VISIT_COOLDOWN_MIN_MS + stableUnsignedHash % PET_VISIT_COOLDOWN_RANGE_MS
}
internal fun reducePetVisitRequestState(
state: PetVisitRequestState,
isStreaming: Boolean,
assistantUiKey: String?,
nowElapsedMs: Long,
completionSettled: Boolean = true,
responseVisitDelayMs: Long? = null,
): PetVisitRequestState {
require(nowElapsedMs >= 0L) { "Elapsed time must be non-negative." }
require(responseVisitDelayMs == null || responseVisitDelayMs > 0L) {
"Response visit delay must be positive when provided."
}
val livePending = state.pending?.takeUnless {
it.readinessAt(nowElapsedMs) == PetVisitReadiness.Expired
}
if (isStreaming) return state.copy(streamArmed = true, pending = livePending)
if (!completionSettled) return state.copy(pending = livePending)
if (!state.streamArmed) return state.copy(pending = livePending)
val settledKey = assistantUiKey?.takeIf(String::isNotBlank)
?: return state.copy(streamArmed = false, pending = livePending)
if (settledKey == state.lastRequestedAssistantUiKey) {
return state.copy(streamArmed = false, pending = livePending)
}
val notBefore = nowElapsedMs + (
responseVisitDelayMs ?: deterministicPetVisitCooldownMs(settledKey)
)
val request = PetVisitRequest(
assistantUiKey = settledKey,
targetKey = "chat-message:$settledKey",
notBeforeElapsedMs = notBefore,
expiresAtElapsedMs = notBefore + PET_VISIT_EXPIRY_WINDOW_MS,
)
return PetVisitRequestState(
streamArmed = false,
lastRequestedAssistantUiKey = settledKey,
pending = request,
)
}
@Stable
class PetCompanionCoordinator {
private var renderState by mutableStateOf(AvatarRenderState(SphereState.Idle))
private var visitRequestState by mutableStateOf(PetVisitRequestState())
private val surfaces = mutableStateMapOf<String, PetCompanionSurface>()
val pendingVisitRequest: PetVisitRequest?
get() = visitRequestState.pending
fun publishRenderState(renderState: AvatarRenderState) {
this.renderState = renderState
}
fun observeChatStream(
isStreaming: Boolean,
assistantUiKey: String?,
completionSettled: Boolean,
nowElapsedMs: Long,
responseVisitDelayMs: Long,
) {
visitRequestState = reducePetVisitRequestState(
state = visitRequestState,
isStreaming = isStreaming,
assistantUiKey = assistantUiKey,
completionSettled = completionSettled,
nowElapsedMs = nowElapsedMs,
responseVisitDelayMs = responseVisitDelayMs,
)
}
/** Remove a request once consumed, superseded, or noticed after expiry. */
fun clearVisitRequest(assistantUiKey: String) {
if (visitRequestState.pending?.assistantUiKey == assistantUiKey) {
visitRequestState = visitRequestState.copy(pending = null)
}
}
fun publishSurface(owner: String, scrolling: Boolean, hidden: Boolean) {
require(owner.isNotBlank()) { "Pet surface owner must not be blank." }
surfaces[owner] = PetCompanionSurface(scrolling, hidden)
}
fun clearSurface(owner: String) {
surfaces.remove(owner)
}
fun activityFor(owner: String?): PetCompanionActivity {
val surface = owner?.let(surfaces::get)
return PetCompanionActivity(
renderState = renderState,
scrolling = surface?.scrolling == true,
hidden = surface?.hidden == true,
)
}
}
private data class PetCompanionSurface(val scrolling: Boolean, val hidden: Boolean)
val LocalPetCompanionCoordinator = staticCompositionLocalOf { PetCompanionCoordinator() }
/**
* Explicit UI surfaces measured by their owners. Their top edges become
* walkable perches, matching Hermes Desktop's live-DOM ledge model without
* inserting layout space or guessing from the semantics tree.
*/
@Stable
class PetSafeAreaRegistry {
// Compatibility view consumed by the current single-rail host.
internal val walkRegions = mutableStateMapOf<String, Rect>()
private val perchRegions = mutableStateMapOf<String, PetMeasuredPerch>()
private val obstacleRegions = mutableStateMapOf<String, PetMeasuredObstacle>()
private val visitTargetRegions = mutableStateMapOf<String, PetMeasuredVisitTarget>()
internal fun updateWalkRegion(key: String, bounds: Rect) {
updatePerch(key, bounds, PetRouteScope())
}
internal fun updatePerch(key: String, bounds: Rect, routeScope: PetRouteScope) {
walkRegions[key] = bounds
perchRegions[key] = PetMeasuredPerch(key, bounds.toPetObstacle(), routeScope)
}
internal fun removeWalkRegion(key: String) {
removePerch(key)
}
internal fun removePerch(key: String) {
walkRegions.remove(key)
perchRegions.remove(key)
}
internal fun updateObstacle(key: String, bounds: Rect, routeScope: PetRouteScope) {
obstacleRegions[key] = PetMeasuredObstacle(key, bounds.toPetObstacle(), routeScope)
}
internal fun removeObstacle(key: String) {
obstacleRegions.remove(key)
}
internal fun updateVisitTarget(key: String, bounds: Rect, routeScope: PetRouteScope) {
visitTargetRegions[key] = PetMeasuredVisitTarget(key, bounds.toPetObstacle(), routeScope)
}
internal fun removeVisitTarget(key: String) {
visitTargetRegions.remove(key)
}
/** Immutable, deterministic view containing only surfaces valid for [route]. */
fun snapshot(route: String?): PetSafeAreaSnapshot = PetSafeAreaSnapshot(
route = route,
perches = perchRegions.values
.filter { it.routeScope.includes(route) }
.sortedBy { it.key },
obstacles = obstacleRegions.values
.filter { it.routeScope.includes(route) }
.sortedBy { it.key },
visitTargets = visitTargetRegions.values
.filter { it.routeScope.includes(route) }
.sortedBy { it.key },
)
private fun Rect.toPetObstacle(): PetObstacle = PetObstacle(left, top, right, bottom)
}
private fun Set<String>.toPetRouteScope(): PetRouteScope = PetRouteScope(toSet())
private fun Modifier.measuredPetSurface(
key: String,
routes: Set<String>,
update: PetSafeAreaRegistry.(String, Rect, PetRouteScope) -> Unit,
remove: PetSafeAreaRegistry.(String) -> Unit,
): Modifier = composed {
require(key.isNotBlank()) { "Pet surface key must not be blank." }
val registry = LocalPetSafeAreaRegistry.current
val routeScope = routes.toPetRouteScope()
DisposableEffect(registry, key, routeScope) {
onDispose { remove.invoke(registry, key) }
}
onGloballyPositioned { coordinates ->
update.invoke(registry, key, coordinates.boundsInRoot(), routeScope)
}
}
val LocalPetSafeAreaRegistry = staticCompositionLocalOf { PetSafeAreaRegistry() }
/** Register an existing UI element whose top edge is a safe pet perch. */
fun Modifier.petPerchSurface(
key: String,
routes: Set<String> = emptySet(),
): Modifier = measuredPetSurface(
key = key,
routes = routes,
update = PetSafeAreaRegistry::updatePerch,
remove = PetSafeAreaRegistry::removePerch,
)
/** Register an existing UI element as a collision obstacle. */
fun Modifier.petObstacleSurface(
key: String,
routes: Set<String> = emptySet(),
): Modifier = measuredPetSurface(
key = key,
routes = routes,
update = PetSafeAreaRegistry::updateObstacle,
remove = PetSafeAreaRegistry::removeObstacle,
)
/**
* Register an existing UI element as a temporary point of interest. Visit
* targets are measured but never promoted to walkable rails or obstacles.
*/
fun Modifier.petVisitTargetSurface(
key: String,
routes: Set<String> = emptySet(),
): Modifier = measuredPetSurface(
key = key,
routes = routes,
update = PetSafeAreaRegistry::updateVisitTarget,
remove = PetSafeAreaRegistry::removeVisitTarget,
)
@@ -0,0 +1,83 @@
package com.hermesandroid.relay.ui.components.pet
enum class PetSuspensionReason {
Keyboard,
Modal,
Voice,
Startup,
Accessibility,
NoSafePosition,
}
sealed interface PetRoamingState {
val home: PetPlacement
data class Docked(override val home: PetPlacement) : PetRoamingState
data class Roaming(
override val home: PetPlacement,
val waypoint: PetPoint,
) : PetRoamingState
data class Dragging(
override val home: PetPlacement,
val pointer: PetPoint,
) : PetRoamingState
data class Suspended(
override val home: PetPlacement,
val reason: PetSuspensionReason,
) : PetRoamingState
}
sealed interface PetRoamingEvent {
data class StartRoaming(val waypoint: PetPoint) : PetRoamingEvent
data class BeginDrag(val pointer: PetPoint) : PetRoamingEvent
data class DragTo(val pointer: PetPoint) : PetRoamingEvent
data class Drop(val placement: PetPlacement) : PetRoamingEvent
data object Dock : PetRoamingEvent
data class Suspend(val reason: PetSuspensionReason) : PetRoamingEvent
data object Resume : PetRoamingEvent
}
/** Pure state reducer. Unrelated or unsafe transitions are stable no-ops. */
fun reducePetRoamingState(
state: PetRoamingState,
event: PetRoamingEvent,
): PetRoamingState = when (event) {
is PetRoamingEvent.StartRoaming -> when (state) {
is PetRoamingState.Docked,
is PetRoamingState.Roaming -> PetRoamingState.Roaming(state.home, event.waypoint)
is PetRoamingState.Dragging,
is PetRoamingState.Suspended -> state
}
is PetRoamingEvent.BeginDrag -> when (state) {
is PetRoamingState.Docked,
is PetRoamingState.Roaming -> PetRoamingState.Dragging(state.home, event.pointer)
is PetRoamingState.Dragging,
is PetRoamingState.Suspended -> state
}
is PetRoamingEvent.DragTo -> when (state) {
is PetRoamingState.Dragging -> state.copy(pointer = event.pointer)
else -> state
}
is PetRoamingEvent.Drop -> when (state) {
is PetRoamingState.Dragging -> PetRoamingState.Docked(event.placement.sanitized())
else -> state
}
PetRoamingEvent.Dock -> when (state) {
is PetRoamingState.Roaming -> PetRoamingState.Docked(state.home)
else -> state
}
is PetRoamingEvent.Suspend -> PetRoamingState.Suspended(state.home, event.reason)
PetRoamingEvent.Resume -> when (state) {
is PetRoamingState.Suspended -> PetRoamingState.Docked(state.home)
else -> state
}
}
@@ -29,10 +29,12 @@ import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.vector.ImageVector
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import com.hermesandroid.relay.R
import com.hermesandroid.relay.ui.theme.HermesRelayTheme
@Composable
@@ -144,8 +146,8 @@ private fun OnboardingPagePreview() {
HermesRelayTheme {
OnboardingPage(
icon = Icons.AutoMirrored.Filled.Chat,
title = "Talk to Your Agent",
description = "Stream conversations with any Hermes profile. Ask questions, run tasks, and collaborate in real time."
title = stringResource(R.string.onboarding_talk_to_agent),
description = stringResource(R.string.onboarding_stream_desc)
)
}
}
@@ -156,11 +158,11 @@ private fun OnboardingPageWithContentPreview() {
HermesRelayTheme {
OnboardingPage(
icon = Icons.AutoMirrored.Filled.Chat,
title = "Let's Connect",
description = "Enter your relay server URL to get started."
title = stringResource(R.string.onboarding_lets_connect),
description = stringResource(R.string.onboarding_server_url)
) {
Text(
text = "Custom content slot",
text = stringResource(R.string.onboarding_custom_slot),
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.primary
)
@@ -41,12 +41,15 @@ import androidx.compose.material3.TextButton
import androidx.compose.material3.TopAppBar
import androidx.compose.material3.TopAppBarDefaults
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.runtime.snapshotFlow
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
@@ -62,6 +65,7 @@ import com.hermesandroid.relay.R
import com.hermesandroid.relay.data.BuildFlavor
import com.hermesandroid.relay.data.FeatureFlags
import com.hermesandroid.relay.ui.components.WhatsNewDialog
import com.hermesandroid.relay.ui.components.pet.LocalPetCompanionCoordinator
import androidx.compose.ui.window.Dialog
import androidx.compose.ui.window.DialogProperties
import com.hermesandroid.relay.ui.theme.gradientBorder
@@ -69,6 +73,7 @@ import com.hermesandroid.relay.update.UpdateCheckResult
import com.hermesandroid.relay.viewmodel.ConnectionViewModel
import com.hermesandroid.relay.viewmodel.UpdateViewModel
import androidx.lifecycle.viewmodel.compose.viewModel
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.launch
/**
@@ -100,6 +105,25 @@ fun AboutScreen(
var showWhatsNew by remember { mutableStateOf(false) }
var showChangelog by remember { mutableStateOf(false) }
val aboutScrollState = rememberScrollState()
val petCompanionCoordinator = LocalPetCompanionCoordinator.current
LaunchedEffect(aboutScrollState, petCompanionCoordinator) {
snapshotFlow {
aboutScrollState.isScrollInProgress to (showWhatsNew || showChangelog)
}
.distinctUntilChanged()
.collect { (scrolling, hidden) ->
petCompanionCoordinator.publishSurface(
owner = "settings/about",
scrolling = scrolling,
hidden = hidden,
)
}
}
DisposableEffect(petCompanionCoordinator) {
onDispose { petCompanionCoordinator.clearSurface("settings/about") }
}
Scaffold(
topBar = {
TopAppBar(
@@ -122,7 +146,7 @@ fun AboutScreen(
modifier = Modifier
.fillMaxSize()
.padding(innerPadding)
.verticalScroll(rememberScrollState())
.verticalScroll(aboutScrollState)
.padding(horizontal = 16.dp, vertical = 16.dp),
verticalArrangement = Arrangement.spacedBy(16.dp),
) {
@@ -51,6 +51,7 @@ import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AppCompatDelegate
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
@@ -59,6 +60,7 @@ import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.snapshotFlow
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
@@ -70,18 +72,23 @@ import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.hermesandroid.relay.R
import com.hermesandroid.relay.data.AppLanguage
import com.hermesandroid.relay.data.MAX_PET_SIZE_SCALE
import com.hermesandroid.relay.data.MIN_PET_SIZE_SCALE
import com.hermesandroid.relay.ui.components.LocalAvailableSphereSkins
import com.hermesandroid.relay.ui.components.SphereRegistry
import com.hermesandroid.relay.ui.components.SphereSkin
import com.hermesandroid.relay.ui.components.SphereSkinSource
import com.hermesandroid.relay.ui.components.SphereState
import com.hermesandroid.relay.ui.components.reactivityLabels
import com.hermesandroid.relay.ui.components.floatingPetDimensions
import com.hermesandroid.relay.ui.components.avatar.AgentAvatar
import com.hermesandroid.relay.ui.components.avatar.AvatarRenderState
import com.hermesandroid.relay.ui.components.avatar.AvatarSource
import com.hermesandroid.relay.ui.components.avatar.LocalAgentAvatar
import com.hermesandroid.relay.ui.components.avatar.LocalAvailableAvatars
import com.hermesandroid.relay.ui.components.avatar.SphereAvatar
import com.hermesandroid.relay.ui.components.avatar.LocalAvailablePets
import com.hermesandroid.relay.ui.components.avatar.LocalFloatingPet
import com.hermesandroid.relay.ui.components.pet.LocalPetCompanionCoordinator
import com.hermesandroid.relay.ui.components.pet.petObstacleSurface
import com.hermesandroid.relay.ui.components.pet.petPerchSurface
import com.hermesandroid.relay.ui.theme.AppFont
import com.hermesandroid.relay.ui.theme.AppTheme
import com.hermesandroid.relay.ui.theme.AppThemes
@@ -90,7 +97,24 @@ import com.hermesandroid.relay.ui.theme.ThemeMode
import com.hermesandroid.relay.ui.theme.gradientBorder
import com.hermesandroid.relay.viewmodel.ConnectionViewModel
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.launch
import kotlin.math.roundToInt
private const val APPEARANCE_PET_SURFACE_ROUTE = "settings/appearance"
private val APPEARANCE_PET_SURFACE_ROUTES = setOf(APPEARANCE_PET_SURFACE_ROUTE)
/** Each card remains a top-edge ledge while its controls stay forbidden terrain. */
private fun Modifier.appearancePetSurface(key: String): Modifier {
val surfaceKey = "appearance-card:$key"
return petPerchSurface(
key = surfaceKey,
routes = APPEARANCE_PET_SURFACE_ROUTES,
).petObstacleSurface(
key = "$surfaceKey:controls",
routes = APPEARANCE_PET_SURFACE_ROUTES,
)
}
/**
* Dedicated Appearance settings screen. Hosts theme picker (auto/light/dark),
@@ -102,6 +126,7 @@ import kotlinx.coroutines.launch
fun AppearanceSettingsScreen(
connectionViewModel: ConnectionViewModel,
onBack: () -> Unit,
onBrowsePetdex: () -> Unit = {},
) {
val theme by connectionViewModel.theme.collectAsState()
val appThemeId by connectionViewModel.appTheme.collectAsState()
@@ -110,6 +135,22 @@ fun AppearanceSettingsScreen(
val snackbarHostState = remember { SnackbarHostState() }
var pendingDelete by remember { mutableStateOf<AgentAvatar?>(null) }
val appearanceScrollState = rememberScrollState()
val petCompanionCoordinator = LocalPetCompanionCoordinator.current
LaunchedEffect(appearanceScrollState, petCompanionCoordinator) {
snapshotFlow { appearanceScrollState.isScrollInProgress to (pendingDelete != null) }
.distinctUntilChanged()
.collect { (scrolling, hidden) ->
petCompanionCoordinator.publishSurface(
owner = APPEARANCE_PET_SURFACE_ROUTE,
scrolling = scrolling,
hidden = hidden,
)
}
}
DisposableEffect(petCompanionCoordinator) {
onDispose { petCompanionCoordinator.clearSurface(APPEARANCE_PET_SURFACE_ROUTE) }
}
val importLauncher = rememberLauncherForActivityResult(
ActivityResultContracts.OpenDocument()
) { uri -> uri?.let { connectionViewModel.importPet(it) } }
@@ -161,7 +202,7 @@ fun AppearanceSettingsScreen(
modifier = Modifier
.fillMaxSize()
.padding(innerPadding)
.verticalScroll(rememberScrollState())
.verticalScroll(appearanceScrollState)
.padding(horizontal = 16.dp, vertical = 16.dp),
verticalArrangement = Arrangement.spacedBy(16.dp),
) {
@@ -174,6 +215,7 @@ fun AppearanceSettingsScreen(
Card(
modifier = Modifier
.appearancePetSurface("theme")
.fillMaxWidth()
.gradientBorder(
shape = RoundedCornerShape(12.dp),
@@ -219,6 +261,7 @@ fun AppearanceSettingsScreen(
Card(
modifier = Modifier
.appearancePetSurface("language")
.fillMaxWidth()
.gradientBorder(
shape = RoundedCornerShape(12.dp),
@@ -249,6 +292,7 @@ fun AppearanceSettingsScreen(
AppLanguage.JAPANESE to stringResource(R.string.appearance_language_japanese),
AppLanguage.SIMPLIFIED_CHINESE to stringResource(R.string.appearance_language_simplified_chinese),
AppLanguage.SPANISH to stringResource(R.string.appearance_language_spanish),
AppLanguage.RUSSIAN to stringResource(R.string.appearance_language_russian),
)
FlowRow(
@@ -289,6 +333,7 @@ fun AppearanceSettingsScreen(
Card(
modifier = Modifier
.appearancePetSurface("display")
.fillMaxWidth()
.gradientBorder(
shape = RoundedCornerShape(12.dp),
@@ -408,6 +453,7 @@ fun AppearanceSettingsScreen(
Card(
modifier = Modifier
.appearancePetSurface("font")
.fillMaxWidth()
.gradientBorder(
shape = RoundedCornerShape(12.dp),
@@ -449,6 +495,7 @@ fun AppearanceSettingsScreen(
Card(
modifier = Modifier
.appearancePetSurface("animation")
.fillMaxWidth()
.gradientBorder(
shape = RoundedCornerShape(12.dp),
@@ -572,17 +619,102 @@ fun AppearanceSettingsScreen(
}
}
// Agent avatar section — choose the avatar first (the built-in sphere
// plus any imported "pets"; add/remove pets in-app below). When the
// sphere avatar is selected its skin chips nest below: avatar → skin.
Text(
text = stringResource(R.string.appearance_agent_avatar),
text = stringResource(R.string.appearance_background_visualization),
style = MaterialTheme.typography.titleMedium,
color = MaterialTheme.colorScheme.primary,
)
Card(
modifier = Modifier
.appearancePetSurface("background")
.fillMaxWidth()
.gradientBorder(
shape = RoundedCornerShape(12.dp),
isDarkTheme = isDarkTheme,
),
colors = CardDefaults.cardColors(
containerColor = MaterialTheme.colorScheme.surfaceVariant,
),
) {
val backgroundVisualizationEnabled by
connectionViewModel.backgroundVisualizationEnabled.collectAsState()
val availableSkins = LocalAvailableSphereSkins.current
val sphereSkinId by connectionViewModel.sphereSkin.collectAsState()
val effectiveSkinId = SphereRegistry.resolve(
selectedId = sphereSkinId,
themeDefaultSkinId = selectedTheme.defaultSphereSkinId,
available = availableSkins,
).id
val brand = LocalBrand.current
Column(
modifier = Modifier.padding(16.dp),
verticalArrangement = Arrangement.spacedBy(12.dp),
) {
Text(
text = stringResource(R.string.appearance_background_visualization_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
FlowRow(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
FilterChip(
selected = !backgroundVisualizationEnabled,
onClick = { connectionViewModel.setBackgroundVisualizationEnabled(false) },
label = { Text(stringResource(R.string.appearance_background_off)) },
)
FilterChip(
selected = backgroundVisualizationEnabled,
onClick = { connectionViewModel.setBackgroundVisualizationEnabled(true) },
label = { Text(stringResource(R.string.appearance_background_sphere)) },
)
}
if (backgroundVisualizationEnabled) {
HorizontalDivider()
Text(
text = stringResource(R.string.appearance_sphere_skin),
style = MaterialTheme.typography.labelLarge,
color = MaterialTheme.colorScheme.onSurface,
)
FlowRow(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(10.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
availableSkins.forEach { skin ->
SphereSkinChip(
skin = skin,
brand = brand,
selected = skin.id == effectiveSkinId,
onClick = { connectionViewModel.setSphereSkin(skin.id) },
)
}
}
Text(
text = stringResource(R.string.appearance_sphere_custom_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
}
// Floating pets are companions, not agent identity or background art.
Text(
text = stringResource(R.string.appearance_floating_pet),
style = MaterialTheme.typography.titleMedium,
color = MaterialTheme.colorScheme.primary
)
Card(
modifier = Modifier
.appearancePetSurface("floating-pet")
.fillMaxWidth()
.gradientBorder(
shape = RoundedCornerShape(12.dp),
@@ -593,22 +725,15 @@ fun AppearanceSettingsScreen(
)
) {
val brand = LocalBrand.current
val availableAvatars = LocalAvailableAvatars.current
val activeAvatar = LocalAgentAvatar.current
val availableSkins = LocalAvailableSphereSkins.current
val sphereSkinId by connectionViewModel.sphereSkin.collectAsState()
val effectiveSkinId = SphereRegistry.resolve(
selectedId = sphereSkinId,
themeDefaultSkinId = selectedTheme.defaultSphereSkinId,
available = availableSkins,
).id
val availablePets = LocalAvailablePets.current
val activePet = LocalFloatingPet.current
Column(
modifier = Modifier.padding(16.dp),
verticalArrangement = Arrangement.spacedBy(12.dp)
) {
Text(
text = stringResource(R.string.appearance_agent_avatar_desc),
text = stringResource(R.string.appearance_floating_pet_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant
)
@@ -618,27 +743,29 @@ fun AppearanceSettingsScreen(
horizontalArrangement = Arrangement.spacedBy(10.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
availableAvatars.forEach { avatar ->
FilterChip(
selected = activePet == null,
onClick = { connectionViewModel.setFloatingPet(null) },
label = { Text(stringResource(R.string.appearance_floating_pet_none)) },
)
availablePets.forEach { pet ->
AgentAvatarChip(
avatar = avatar,
avatar = pet,
brand = brand,
selected = avatar.id == activeAvatar.id,
// Persist + switch. RelayApp re-resolves
// LocalAgentAvatar from this pref, so every
// surface (and these chips) updates.
onClick = { connectionViewModel.setAgentAvatar(avatar.id) },
selected = pet.id == activePet?.id,
onClick = { connectionViewModel.setFloatingPet(pet.id) },
)
}
}
// Add / manage user pets in-app — the reliable alternative to
// adb push (scoped storage blocks or stalls it on many devices).
val userAvatars = availableAvatars.filter { it.source == AvatarSource.USER }
val userAvatars = availablePets.filter { it.source == AvatarSource.USER }
Row(
FlowRow(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically,
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
OutlinedButton(
onClick = {
@@ -660,6 +787,9 @@ fun AppearanceSettingsScreen(
TextButton(onClick = { connectionViewModel.refreshAgentAvatars() }) {
Text(stringResource(R.string.appearance_rescan))
}
TextButton(onClick = onBrowsePetdex) {
Text(stringResource(R.string.appearance_browse_petdex))
}
}
Text(
@@ -699,10 +829,13 @@ fun AppearanceSettingsScreen(
// Pet playback-speed tuning (selected pet only) — scales the
// authored fps live, no re-authoring or re-importing needed.
if (activeAvatar.source == AvatarSource.USER) {
if (activePet?.source == AvatarSource.USER) {
HorizontalDivider()
val petSpeed by connectionViewModel.petSpeed.collectAsState()
val petSizeScale by connectionViewModel.petSizeScale.collectAsState()
val petRoamingEnabled by connectionViewModel.petRoamingEnabled.collectAsState()
val petTemperament by connectionViewModel.petTemperament.collectAsState()
Text(
text = stringResource(R.string.appearance_playback_speed, "%.1f".format(java.util.Locale.US, petSpeed)),
style = MaterialTheme.typography.labelLarge,
@@ -720,6 +853,26 @@ fun AppearanceSettingsScreen(
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
Text(
text = stringResource(
R.string.appearance_pet_size,
(petSizeScale * 100f).roundToInt(),
),
style = MaterialTheme.typography.labelLarge,
color = MaterialTheme.colorScheme.onSurface,
)
Slider(
value = petSizeScale,
onValueChange = connectionViewModel::setPetSizeScale,
valueRange = MIN_PET_SIZE_SCALE..MAX_PET_SIZE_SCALE,
steps = 5,
)
Text(
text = stringResource(R.string.appearance_pet_size_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.SpaceBetween,
@@ -743,6 +896,68 @@ fun AppearanceSettingsScreen(
)
}
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = Alignment.CenterVertically,
) {
Column(modifier = Modifier.weight(1f)) {
Text(
text = stringResource(R.string.appearance_pet_roaming),
style = MaterialTheme.typography.bodyMedium,
)
Text(
text = stringResource(R.string.appearance_pet_roaming_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
Switch(
checked = petRoamingEnabled,
onCheckedChange = { connectionViewModel.setPetRoamingEnabled(it) },
)
}
Text(
text = stringResource(R.string.appearance_pet_temperament),
style = MaterialTheme.typography.labelLarge,
color = MaterialTheme.colorScheme.onSurface,
)
Text(
text = stringResource(R.string.appearance_pet_temperament_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
FlowRow(
modifier = Modifier
.fillMaxWidth()
.alpha(if (petRoamingEnabled) 1f else 0.6f),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
petTemperamentOptions.forEach { option ->
FilterChip(
selected = option.temperament == petTemperament,
onClick = {
connectionViewModel.setPetTemperament(option.temperament)
},
enabled = petRoamingEnabled,
label = { Text(stringResource(option.labelRes)) },
)
}
}
Text(
text = stringResource(
petTemperamentOption(petTemperament).descriptionRes,
),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
modifier = Modifier.alpha(if (petRoamingEnabled) 1f else 0.6f),
)
TextButton(onClick = connectionViewModel::resetPetPlacement) {
Text(stringResource(R.string.floating_pet_action_reset))
}
// Live state preview — drive the pet through each state to
// verify look/speed/stabilization without running the agent.
HorizontalDivider()
@@ -777,9 +992,16 @@ fun AppearanceSettingsScreen(
.background(MaterialTheme.colorScheme.surface),
contentAlignment = Alignment.Center,
) {
Box(modifier = Modifier.size(140.dp)) {
Box(
modifier = Modifier.size(
floatingPetDimensions(
compact = false,
sizeScale = petSizeScale,
).visualSizeDp.dp,
),
) {
key(greetKey) {
activeAvatar.Render(
activePet.Render(
state = AvatarRenderState(
state = previewSphereState,
toolCallBurst = previewBurst,
@@ -832,40 +1054,6 @@ fun AppearanceSettingsScreen(
)
}
// Second level of the model: skin chips, shown only when the
// sphere avatar is active (a pet carries no skins).
if (activeAvatar.id == SphereAvatar.id) {
HorizontalDivider()
Text(
text = stringResource(R.string.appearance_sphere_skin),
style = MaterialTheme.typography.labelLarge,
color = MaterialTheme.colorScheme.onSurface
)
FlowRow(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(10.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
availableSkins.forEach { skin ->
SphereSkinChip(
skin = skin,
brand = brand,
selected = skin.id == effectiveSkinId,
onClick = { connectionViewModel.setSphereSkin(skin.id) },
)
}
}
HorizontalDivider()
Text(
text = stringResource(R.string.appearance_sphere_custom_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant
)
}
}
}
}
@@ -24,6 +24,7 @@ import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.consumeWindowInsets
import androidx.compose.foundation.layout.ime
import androidx.compose.foundation.layout.imePadding
import androidx.compose.foundation.layout.navigationBars
import androidx.compose.foundation.layout.navigationBarsPadding
@@ -47,6 +48,7 @@ import androidx.compose.material.icons.filled.AutoAwesome
import androidx.compose.material.icons.filled.Bolt
import androidx.compose.material.icons.filled.ChatBubble
import androidx.compose.material.icons.filled.Code
import androidx.compose.material.icons.filled.ContentCopy
import androidx.compose.material.icons.filled.Edit
import androidx.compose.material.icons.filled.Menu
import androidx.compose.material.icons.filled.MoreVert
@@ -102,8 +104,10 @@ import androidx.compose.ui.graphics.Color
import androidx.compose.ui.hapticfeedback.HapticFeedbackType
import androidx.compose.ui.platform.ClipEntry
import androidx.compose.ui.platform.LocalClipboard
import androidx.compose.ui.platform.LocalConfiguration
import androidx.compose.ui.platform.LocalFocusManager
import androidx.compose.ui.platform.LocalHapticFeedback
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.res.pluralStringResource
import androidx.compose.ui.res.stringResource
@@ -139,6 +143,7 @@ import androidx.compose.material3.SmallFloatingActionButton
import androidx.compose.material3.SnackbarHost
import androidx.compose.material3.SnackbarHostState
import androidx.compose.material3.SnackbarDuration
import androidx.compose.material3.SnackbarResult
import android.content.ClipData
import android.content.Intent
import android.net.Uri
@@ -156,13 +161,16 @@ import com.hermesandroid.relay.data.AgentDisplay
import com.hermesandroid.relay.data.Attachment
import com.hermesandroid.relay.data.ChatMessage
import com.hermesandroid.relay.data.Connection
import com.hermesandroid.relay.data.HermesCardAction
import com.hermesandroid.relay.data.MessageRole
import com.hermesandroid.relay.data.VoicePresentationMode
import com.hermesandroid.relay.data.hermesProcessNotificationOrNull
import com.hermesandroid.relay.ui.components.AgentInfoSheet
import com.hermesandroid.relay.ui.components.BackgroundTaskCard
import com.hermesandroid.relay.ui.components.LocalRelayServerImageResolver
import com.hermesandroid.relay.ui.components.RelayServerImageResolver
import com.hermesandroid.relay.ui.components.ChatInputBar
import com.hermesandroid.relay.ui.components.ConversationVoiceDock
import com.hermesandroid.relay.ui.components.CleanChatMode
import com.hermesandroid.relay.ui.components.ChatInputPickerControl
import com.hermesandroid.relay.ui.components.ChatInputPickerOption
@@ -174,18 +182,29 @@ import com.hermesandroid.relay.ui.components.ConnectionStatusBadge
import com.hermesandroid.relay.ui.components.CommandRow
import com.hermesandroid.relay.ui.components.CompactToolCall
import com.hermesandroid.relay.ui.components.ContextMeterBar
import com.hermesandroid.relay.ui.components.CHAT_PET_WALK_REGION
import com.hermesandroid.relay.ui.components.CHAT_PET_ASSISTANT_MESSAGE_PERCH_PREFIX
import com.hermesandroid.relay.ui.components.CHAT_PET_STEP_MESSAGE_MARKER
import com.hermesandroid.relay.ui.components.CHAT_PET_USER_MESSAGE_PERCH_PREFIX
import com.hermesandroid.relay.ui.components.GatewayBackgroundProcessSheet
import com.hermesandroid.relay.ui.components.GatewayBackgroundProcessStrip
import com.hermesandroid.relay.ui.components.InjectedContextSheet
import com.hermesandroid.relay.ui.components.InlineAutocomplete
import com.hermesandroid.relay.ui.components.loadedContentTransform
import com.hermesandroid.relay.ui.components.MessageBubble
import com.hermesandroid.relay.ui.components.newestPetPerchUiKey
import com.hermesandroid.relay.ui.components.newestPetVisitTargetUiKey
import com.hermesandroid.relay.ui.components.petPerchUiKeys
import com.hermesandroid.relay.ui.components.SyntheticProcessNotificationNotice
import com.hermesandroid.relay.ui.components.avatar.AvatarRenderState
import androidx.compose.ui.layout.ContentScale
import coil3.compose.AsyncImage
import com.hermesandroid.relay.ui.components.LocalAgentIconPath
import com.hermesandroid.relay.ui.components.avatar.LocalAgentAvatar
import com.hermesandroid.relay.ui.components.avatar.LocalBackgroundVisualizationEnabled
import com.hermesandroid.relay.ui.components.pet.LocalPetCompanionCoordinator
import com.hermesandroid.relay.ui.components.pet.petObstacleSurface
import com.hermesandroid.relay.ui.components.pet.petPerchSurface
import java.io.File
import com.hermesandroid.relay.ui.components.RelayChromeIconButton
import com.hermesandroid.relay.ui.components.SphereState
@@ -204,6 +223,7 @@ import com.hermesandroid.relay.ui.components.showsImageGenerationPlaceholder
import com.hermesandroid.relay.ui.components.VoiceModeOverlay
import com.hermesandroid.relay.ui.LocalSnackbarHost
import com.hermesandroid.relay.ui.showHumanError
import com.hermesandroid.relay.util.HumanErrorAction
import com.hermesandroid.relay.ui.theme.RelayRefresh
import kotlin.math.abs
import com.hermesandroid.relay.ui.theme.relayGridTexture
@@ -217,6 +237,8 @@ import com.hermesandroid.relay.viewmodel.ChatTransportReadiness
import com.hermesandroid.relay.viewmodel.ConnectionViewModel
import com.hermesandroid.relay.viewmodel.resolveChatRuntimeStatus
import com.hermesandroid.relay.viewmodel.VoiceViewModel
import com.hermesandroid.relay.viewmodel.VoiceState
import com.hermesandroid.relay.assistant.AssistantAppSessionState
import com.hermesandroid.relay.voice.VoiceOverlayHost
import com.hermesandroid.relay.voice.VoiceOverlaySession
import com.hermesandroid.relay.voice.openHermesFromOverlay
@@ -227,6 +249,9 @@ import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
private const val DEFAULT_CHAR_LIMIT = 4096
private const val CHAT_SCROLL_TO_BOTTOM_PET_PERCH = "chat-scroll-to-bottom-perch"
private const val CHAT_SCROLL_TO_BOTTOM_PET_OBSTACLE = "chat-scroll-to-bottom-obstacle"
private val CHAT_PET_ROUTES = setOf("chat")
internal fun resolveChatHeaderSubtitle(
isStreaming: Boolean,
@@ -280,6 +305,17 @@ internal fun ChatScrollSnapshot.isCompletionAfter(previous: ChatScrollSnapshot?)
previous.messageCount == messageCount &&
previous.lastMessageUiKey == lastMessageUiKey
internal fun releaseRetainedLiveTail(
retainedUiKey: String?,
completedUiKey: String?,
): String? = retainedUiKey?.takeUnless { it == completedUiKey }
internal fun tailEndScrollOffset(
tailSizePx: Int,
footerSizePx: Int,
viewportSizePx: Int,
): Int = (tailSizePx + footerSizePx - viewportSizePx).coerceAtLeast(0)
private class ChatTailTransitionRef(
var snapshot: ChatScrollSnapshot? = null,
)
@@ -439,6 +475,8 @@ fun ChatScreen(
voiceViewModel: VoiceViewModel,
voiceClient: RelayVoiceClient? = null,
maxBubbleWidth: Dp = 300.dp,
voicePresentationMode: VoicePresentationMode = VoicePresentationMode.Focus,
onVoicePresentationModeChange: (VoicePresentationMode) -> Unit = {},
// Deep-link nudge from Settings → Active Agent card: when `true`, the
// AgentInfoSheet auto-opens on first composition and [onAgentSheetArgConsumed]
// fires so the host can clear the nav arg (prevents re-open on tab
@@ -451,6 +489,7 @@ fun ChatScreen(
// don't wire navigation.
onNavigateToConnections: () -> Unit = {},
onNavigateToConnect: () -> Unit = onNavigateToConnections,
onRepairConnection: () -> Unit = onNavigateToConnect,
// Offline demo entry, surfaced on the empty-chat "needs connection" card so a
// skipped / never-connected first run can explore without a server. null hides it.
onTryDemo: (() -> Unit)? = null,
@@ -458,6 +497,7 @@ fun ChatScreen(
onNavigateToBridge: () -> Unit = {},
onNavigateToTerminal: () -> Unit = {},
onNavigateToSettings: () -> Unit = {},
onNavigateToAppearanceSettings: () -> Unit = {},
// Voice mode gear button → full Voice Settings screen. Default no-op so
// existing test/preview call sites keep compiling.
onNavigateToVoiceSettings: () -> Unit = {},
@@ -465,19 +505,42 @@ fun ChatScreen(
) {
val voiceUiState by voiceViewModel.uiState.collectAsState()
val isDemoMode by connectionViewModel.isDemoMode.collectAsState()
var voiceCompactMode by remember { mutableStateOf(false) }
var voicePresentationOverride by remember { mutableStateOf<VoicePresentationMode?>(null) }
val effectiveVoicePresentationMode = voicePresentationOverride ?: voicePresentationMode
val conversationVoiceResponseActionsEnabled =
voiceUiState.voiceMode &&
effectiveVoicePresentationMode == VoicePresentationMode.Conversation &&
voiceUiState.state == VoiceState.Idle
val conversationVoiceDockVisible =
voiceUiState.voiceMode &&
effectiveVoicePresentationMode == VoicePresentationMode.Conversation
val setVoicePresentationMode: (VoicePresentationMode) -> Unit = { mode ->
voicePresentationOverride = mode
onVoicePresentationModeChange(mode)
}
val chatAlpha by animateFloatAsState(
targetValue = if (voiceUiState.voiceMode && !voiceCompactMode) 0.4f else 1f,
targetValue = if (
voiceUiState.voiceMode && effectiveVoicePresentationMode == VoicePresentationMode.Focus
) 0.4f else 1f,
animationSpec = tween(300),
label = "chatAlpha",
)
LaunchedEffect(voiceUiState.voiceMode) {
if (!voiceUiState.voiceMode) voicePresentationOverride = null
}
// Route classified chat errors (media cache, streaming failures, …) to
// the app-wide snackbar. Same pattern every VM-bound screen uses.
val snackbarHost = LocalSnackbarHost.current
LaunchedEffect(chatViewModel) {
chatViewModel.errorEvents.collect { err ->
snackbarHost.showHumanError(err)
val result = snackbarHost.showHumanError(err)
if (
result == SnackbarResult.ActionPerformed &&
err.action == HumanErrorAction.Repair
) {
onRepairConnection()
}
}
}
@@ -487,6 +550,7 @@ fun ChatScreen(
val context = LocalContext.current
val lifecycleOwner = LocalLifecycleOwner.current
val voiceOverlayHost = remember { VoiceOverlayHost.install(context) }
val assistantSessionActive by AssistantAppSessionState.active.collectAsState()
var pendingVoiceEnter by remember { mutableStateOf(false) }
var micPermissionDenied by remember { mutableStateOf(false) }
var pendingVoiceOverlayPermission by remember { mutableStateOf(false) }
@@ -497,12 +561,15 @@ fun ChatScreen(
micPermissionDenied = false
if (pendingVoiceEnter && !isDemoMode) {
pendingVoiceEnter = false
setVoicePresentationMode(VoicePresentationMode.Conversation)
com.hermesandroid.relay.wake.WakeWordForegroundService.prepareForVoice()
voiceViewModel.enterVoiceMode()
} else {
pendingVoiceEnter = false
}
} else {
pendingVoiceEnter = false
voicePresentationOverride = null
micPermissionDenied = true
}
}
@@ -513,6 +580,8 @@ fun ChatScreen(
) == android.content.pm.PackageManager.PERMISSION_GRANTED
if (granted) {
micPermissionDenied = false
setVoicePresentationMode(VoicePresentationMode.Conversation)
com.hermesandroid.relay.wake.WakeWordForegroundService.prepareForVoice()
voiceViewModel.enterVoiceMode()
} else {
pendingVoiceEnter = true
@@ -772,9 +841,27 @@ fun ChatScreen(
}
}
val listState = rememberLazyListState()
// Publish only surface-local visibility signals. Live turn state is owned at
// the app root so navigation cannot reset an in-flight companion to Idle.
val petCompanionCoordinator = LocalPetCompanionCoordinator.current
LaunchedEffect(listState, petCompanionCoordinator) {
snapshotFlow { listState.isScrollInProgress to ambientMode }
.distinctUntilChanged()
.collect { (scrolling, hidden) ->
petCompanionCoordinator.publishSurface(
owner = "chat",
scrolling = scrolling,
hidden = hidden,
)
}
}
DisposableEffect(petCompanionCoordinator) {
onDispose { petCompanionCoordinator.clearSurface("chat") }
}
val drawerState = rememberDrawerState(DrawerValue.Closed)
val scope = rememberCoroutineScope()
val copiedToClipboardMsg = stringResource(R.string.chat_copied_to_clipboard)
val copySessionIdLabel = stringResource(R.string.chat_copy_session_id)
val hermesMessageLabel = stringResource(R.string.chat_hermes_message)
val focusManager = LocalFocusManager.current
val finishSuccessfulSend: () -> Unit = {
@@ -788,6 +875,25 @@ fun ChatScreen(
val clipboard = LocalClipboard.current
val haptic = LocalHapticFeedback.current
val snackbarHostState = remember { SnackbarHostState() }
val handleCardAction: (String, String, HermesCardAction) -> Unit =
remember(chatViewModel, context) {
{ messageId, cardKey, action ->
if (action.mode == HermesCardAction.Modes.OPEN_URL) {
chatViewModel.dispatchCardAction(messageId, cardKey, action)
com.hermesandroid.relay.ui.components.handleCardActionExternally(
context,
action,
)
} else {
chatViewModel.dispatchCardAction(messageId, cardKey, action)
}
}
}
val handleCardInput: (String, String, String) -> Unit = remember(chatViewModel) {
{ messageId, cardKey, value ->
chatViewModel.answerAsk(messageId, cardKey, value)
}
}
// Ephemeral notices from the VM (model-switch warnings/errors, etc.) →
// transient snackbar, never a chat bubble.
@@ -827,7 +933,9 @@ fun ChatScreen(
}
val showVoiceSystemOverlay: () -> Unit = {
if (!voiceOverlayHost.hasOverlayPermission()) {
if (assistantSessionActive) {
voiceOverlayHost.hide()
} else if (!voiceOverlayHost.hasOverlayPermission()) {
pendingVoiceOverlayPermission = true
runCatching {
val intent = Intent(
@@ -885,7 +993,13 @@ fun ChatScreen(
if (!voiceUiState.voiceMode) {
voiceOverlayHost.hide()
pendingVoiceOverlayPermission = false
voiceCompactMode = false
}
}
LaunchedEffect(assistantSessionActive) {
if (assistantSessionActive) {
voiceOverlayHost.hide()
pendingVoiceOverlayPermission = false
}
}
@@ -1110,6 +1224,29 @@ fun ChatScreen(
}
}
}
var voiceDockAnchorGuardReady by remember { mutableStateOf(false) }
LaunchedEffect(conversationVoiceDockVisible) {
if (!voiceDockAnchorGuardReady) {
voiceDockAnchorGuardReady = true
return@LaunchedEffect
}
if (messages.isEmpty()) return@LaunchedEffect
// The dock expands inside the composer for 240 ms. Keep the transcript's
// leading visible row fixed while that changes the LazyColumn viewport;
// otherwise a conversation parked at the bottom is clamped forward on
// every animation frame and appears to scroll when voice mode opens.
val anchorIndex = listState.firstVisibleItemIndex
val anchorOffset = listState.firstVisibleItemScrollOffset
var firstFrameNanos = 0L
var frameNanos: Long
do {
if (isUserDragging) return@LaunchedEffect
listState.requestScrollToItem(anchorIndex, anchorOffset)
frameNanos = withFrameNanos { it }
if (firstFrameNanos == 0L) firstFrameNanos = frameNanos
} while (frameNanos - firstFrameNanos < 300_000_000L)
}
// Reaching the bottom by any means (user, follow-pin, content shrank)
// always re-arms auto-follow.
LaunchedEffect(isAtBottom) {
@@ -1130,15 +1267,49 @@ fun ChatScreen(
derivedStateOf {
val retainingVisibleTail = retainedLiveTailUiKey != null &&
messages.lastOrNull()?.uiKey == retainedLiveTailUiKey
val settlingVisibleTail = completionSettlingUiKey != null &&
messages.lastOrNull()?.uiKey == completionSettlingUiKey
messages.isNotEmpty() &&
!isAtBottom &&
!programmaticBottomScroll &&
!((isStreaming || retainingVisibleTail) &&
!((isStreaming || retainingVisibleTail || settlingVisibleTail) &&
smoothAutoScroll &&
!userScrolledAway)
}
}
// Slash command descriptions (resolved in composable scope — the
// remember/derivedStateOf block below is NOT composable, so stringResource
// must be called here, not inside it).
val slashDescNew = stringResource(R.string.slash_new_session)
val slashDescRetry = stringResource(R.string.slash_retry_last)
val slashDescUndo = stringResource(R.string.slash_remove_exchange)
val slashDescTitle = stringResource(R.string.slash_set_title)
val slashDescCompress = stringResource(R.string.slash_compress)
val slashDescRollback = stringResource(R.string.slash_checkpoints)
val slashDescStop = stringResource(R.string.slash_kill_bg)
val slashDescResume = stringResource(R.string.slash_resume)
val slashDescBackground = stringResource(R.string.slash_background_prompt)
val slashDescBtw = stringResource(R.string.slash_side_question)
val slashDescQueue = stringResource(R.string.slash_queue_prompt)
val slashDescApprove = stringResource(R.string.slash_approve)
val slashDescDeny = stringResource(R.string.slash_deny)
val slashDescModel = stringResource(R.string.slash_switch_model)
val slashDescProvider = stringResource(R.string.slash_providers)
val slashDescPersonality = stringResource(R.string.slash_personality)
val slashDescVerbose = stringResource(R.string.slash_tool_progress)
val slashDescYolo = stringResource(R.string.slash_auto_approve)
val slashDescReasoning = stringResource(R.string.slash_reasoning)
val slashDescVoice = stringResource(R.string.slash_voice_mode)
val slashDescReloadMcp = stringResource(R.string.slash_reload_mcp)
val slashDescHelp = stringResource(R.string.slash_commands)
val slashDescStatus = stringResource(R.string.slash_session_info)
val slashDescUsage = stringResource(R.string.slash_token_usage)
val slashDescInsights = stringResource(R.string.slash_analytics)
val slashDescCommands = stringResource(R.string.slash_browse)
val slashDescProfile = stringResource(R.string.slash_active_profile)
val slashDescClearPersonality = stringResource(R.string.slash_clear_personality)
// Build all commands dynamically: built-in + personalities + server
// skills + (gateway) the server's commands.catalog
val allCommands by remember(availableSkills, personalityNames, serverCommands) {
@@ -1147,36 +1318,36 @@ fun ChatScreen(
// Only includes commands available via gateway (not cli_only)
val builtIn = listOf(
// Session
SlashCommand("/new", "Start a new session", "session"),
SlashCommand("/retry", "Retry the last message", "session"),
SlashCommand("/undo", "Remove the last exchange", "session"),
SlashCommand("/title", "Set a title for this session", "session"),
SlashCommand("/new", slashDescNew, "session"),
SlashCommand("/retry", slashDescRetry, "session"),
SlashCommand("/undo", slashDescUndo, "session"),
SlashCommand("/title", slashDescTitle, "session"),
SlashCommand("/branch", "Branch/fork the current session", "session"),
SlashCommand("/compress", "Compress conversation context", "session"),
SlashCommand("/rollback", "List or restore checkpoints", "session"),
SlashCommand("/stop", "Kill running background processes", "session"),
SlashCommand("/resume", "Resume a previous session", "session"),
SlashCommand("/background", "Run a prompt in the background", "session"),
SlashCommand("/btw", "Side question using session context", "session"),
SlashCommand("/queue", "Queue a prompt for the next turn", "session"),
SlashCommand("/approve", "Approve a pending command", "session"),
SlashCommand("/deny", "Deny a pending command", "session"),
SlashCommand("/compress", slashDescCompress, "session"),
SlashCommand("/rollback", slashDescRollback, "session"),
SlashCommand("/stop", slashDescStop, "session"),
SlashCommand("/resume", slashDescResume, "session"),
SlashCommand("/background", slashDescBackground, "session"),
SlashCommand("/btw", slashDescBtw, "session"),
SlashCommand("/queue", slashDescQueue, "session"),
SlashCommand("/approve", slashDescApprove, "session"),
SlashCommand("/deny", slashDescDeny, "session"),
// Configuration
SlashCommand("/model", "Switch model for this session", "configuration"),
SlashCommand("/provider", "Show available providers", "configuration"),
SlashCommand("/personality", "Set a predefined personality", "configuration"),
SlashCommand("/verbose", "Cycle tool progress display", "configuration"),
SlashCommand("/yolo", "Toggle auto-approve mode", "configuration"),
SlashCommand("/reasoning", "Set reasoning effort level", "configuration"),
SlashCommand("/voice", "Toggle voice mode", "configuration"),
SlashCommand("/reload-mcp", "Reload MCP servers", "configuration"),
SlashCommand("/model", slashDescModel, "configuration"),
SlashCommand("/provider", slashDescProvider, "configuration"),
SlashCommand("/personality", slashDescPersonality, "configuration"),
SlashCommand("/verbose", slashDescVerbose, "configuration"),
SlashCommand("/yolo", slashDescYolo, "configuration"),
SlashCommand("/reasoning", slashDescReasoning, "configuration"),
SlashCommand("/voice", slashDescVoice, "configuration"),
SlashCommand("/reload-mcp", slashDescReloadMcp, "configuration"),
// Info
SlashCommand("/help", "Show available commands", "info"),
SlashCommand("/status", "Show session info", "info"),
SlashCommand("/usage", "Show token usage", "info"),
SlashCommand("/insights", "Usage analytics", "info"),
SlashCommand("/commands", "Browse all commands", "info"),
SlashCommand("/profile", "Show active profile", "info"),
SlashCommand("/help", slashDescHelp, "info"),
SlashCommand("/status", slashDescStatus, "info"),
SlashCommand("/usage", slashDescUsage, "info"),
SlashCommand("/insights", slashDescInsights, "info"),
SlashCommand("/commands", slashDescCommands, "info"),
SlashCommand("/profile", slashDescProfile, "info"),
)
// Dynamic personality commands from server, plus the upstream
@@ -1184,7 +1355,7 @@ fun ChatScreen(
val personalities = listOf(
SlashCommand(
command = "/personality none",
description = "Clear the personality overlay",
description = slashDescClearPersonality,
category = "personality"
)
) + personalityNames.map { name ->
@@ -1331,34 +1502,99 @@ fun ChatScreen(
) {
val settlingKey = completionSettlingUiKey ?: return@LaunchedEffect
if (!smoothAutoScroll || userScrolledAway || isUserDragging) {
// Retention is only a completion-transition aid. Never leave the
// finalized tail on the plain streaming renderer just because the
// user disabled follow-scroll or is reading above the bottom.
retainedLiveTailUiKey = releaseRetainedLiveTail(
retainedUiKey = retainedLiveTailUiKey,
completedUiKey = settlingKey,
)
completionSettlingUiKey = null
return@LaunchedEffect
}
var settledFrames = 0
repeat(6) {
var previousMarkdownTailSize: Int? = null
var previousMarkdownFooterSize: Int? = null
val markdownWasAlreadyReleased = retainedLiveTailUiKey != settlingKey
repeat(60) completionFrame@{
withFrameNanos { }
if (messages.lastOrNull()?.uiKey != settlingKey) {
completionSettlingUiKey = null
return@LaunchedEffect
}
if (listState.canScrollForward) {
settledFrames = 0
val viewportHeight = listState.layoutInfo.viewportSize.height
if (viewportHeight > 0) {
listState.scroll(MutatePriority.Default) {
scrollBy(viewportHeight.toFloat())
if (!markdownWasAlreadyReleased && retainedLiveTailUiKey == settlingKey) {
if (listState.canScrollForward) {
settledFrames = 0
val viewportHeight = listState.layoutInfo.viewportSize.height
if (viewportHeight > 0) {
listState.scroll(MutatePriority.Default) {
scrollBy(viewportHeight.toFloat())
}
}
return@completionFrame
}
} else {
settledFrames += 1
if (settledFrames >= 2) {
completionSettlingUiKey = null
return@LaunchedEffect
}
if (settledFrames < 2) return@completionFrame
retainedLiveTailUiKey = releaseRetainedLiveTail(
retainedUiKey = retainedLiveTailUiKey,
completedUiKey = settlingKey,
)
settledFrames = 0
return@completionFrame
}
// Once Markdown owns the row, position its measured trailing edge
// explicitly. `canScrollForward` is insufficient here: LazyColumn
// may preserve the leading edge of a tall item while reporting an
// otherwise valid item anchor. Repeating catches deferred parsing,
// highlighted code, and attachment measurement without competing
// with the ordinary streaming-growth coroutine.
val layout = listState.layoutInfo
val tailIndex = messages.size // header item + zero-based messages
val footerIndex = tailIndex + 1
val tailInfo = layout.visibleItemsInfo.firstOrNull { it.index == tailIndex }
val footerInfo = layout.visibleItemsInfo.firstOrNull { it.index == footerIndex }
if (tailInfo == null) {
listState.scrollToItem(tailIndex)
settledFrames = 0
return@completionFrame
}
val viewportHeight = layout.viewportSize.height
if (viewportHeight <= 0) return@completionFrame
val desiredOffset = tailEndScrollOffset(
tailSizePx = tailInfo.size,
footerSizePx = footerInfo?.size ?: 0,
viewportSizePx = viewportHeight,
)
if (desiredOffset == 0) {
listState.scrollToItem(footerIndex)
} else {
listState.scrollToItem(tailIndex, desiredOffset)
}
val footerSize = footerInfo?.size ?: 0
settledFrames = if (
previousMarkdownTailSize == tailInfo.size &&
previousMarkdownFooterSize == footerSize
) {
settledFrames + 1
} else {
0
}
previousMarkdownTailSize = tailInfo.size
previousMarkdownFooterSize = footerSize
if (settledFrames >= 12) {
completionSettlingUiKey = null
return@LaunchedEffect
}
}
retainedLiveTailUiKey = releaseRetainedLiveTail(
retainedUiKey = retainedLiveTailUiKey,
completedUiKey = settlingKey,
)
completionSettlingUiKey = null
}
@@ -1510,6 +1746,18 @@ fun ChatScreen(
onRenameSession = { sessionId, title ->
chatViewModel.renameSession(sessionId, title)
},
onCopySessionId = { sessionId ->
scope.launch {
clipboard.setClipEntry(
ClipEntry(ClipData.newPlainText(copySessionIdLabel, sessionId))
)
Toast.makeText(
context,
copiedToClipboardMsg,
Toast.LENGTH_SHORT,
).show()
}
},
threadsCapabilityActive = threadsCapabilityActive,
onNewThread = { name ->
chatViewModel.startNewThread(name)
@@ -1829,8 +2077,9 @@ fun ChatScreen(
// once there's a conversation), so it folds into a ⋮
// overflow instead of competing for width with Terminal +
// Settings — which is what was squeezing the title subtitle.
// The overflow only appears when there's something to share.
if (messages.isNotEmpty()) {
// Session identity is useful before the first message; sharing only appears
// once the conversation has content.
if (messages.isNotEmpty() || !currentSessionId.isNullOrBlank()) {
var showOverflowMenu by remember { mutableStateOf(false) }
Box {
RelayChromeIconButton(
@@ -1843,19 +2092,46 @@ fun ChatScreen(
expanded = showOverflowMenu,
onDismissRequest = { showOverflowMenu = false },
) {
DropdownMenuItem(
text = { Text(stringResource(R.string.chat_share_conversation)) },
leadingIcon = {
Icon(
imageVector = Icons.Filled.Share,
contentDescription = null,
)
},
onClick = {
showOverflowMenu = false
shareConversation(context, messages)
},
)
currentSessionId?.takeIf { it.isNotBlank() }?.let { sessionId ->
DropdownMenuItem(
text = { Text(copySessionIdLabel) },
leadingIcon = {
Icon(Icons.Filled.ContentCopy, contentDescription = null)
},
onClick = {
showOverflowMenu = false
scope.launch {
clipboard.setClipEntry(
ClipEntry(
ClipData.newPlainText(
copySessionIdLabel,
sessionId,
)
)
)
snackbarHostState.showSnackbar(
message = copiedToClipboardMsg,
duration = SnackbarDuration.Short,
)
}
},
)
}
if (messages.isNotEmpty()) {
DropdownMenuItem(
text = { Text(stringResource(R.string.chat_share_conversation)) },
leadingIcon = {
Icon(
imageVector = Icons.Filled.Share,
contentDescription = null,
)
},
onClick = {
showOverflowMenu = false
shareConversation(context, messages)
},
)
}
}
}
}
@@ -2006,7 +2282,7 @@ fun ChatScreen(
Spacer(modifier = Modifier.weight(0.15f))
// ASCII sphere (constrained to square aspect)
if (animationEnabled) {
if (LocalBackgroundVisualizationEnabled.current) {
Box(
modifier = Modifier
.fillMaxWidth()
@@ -2018,6 +2294,7 @@ fun ChatScreen(
state = if (error != null) SphereState.Error else SphereState.Idle,
intensity = streamingIntensity,
toolCallBurst = toolCallBurst,
paused = !animationEnabled,
),
modifier = Modifier.fillMaxSize(),
)
@@ -2166,12 +2443,17 @@ fun ChatScreen(
}
) {
// Ambient avatar behind messages
if (animationEnabled && animationBehindChat && !ambientMode) {
if (
LocalBackgroundVisualizationEnabled.current &&
animationBehindChat &&
!ambientMode
) {
LocalAgentAvatar.current.Render(
state = AvatarRenderState(
state = sphereState,
intensity = streamingIntensity,
toolCallBurst = toolCallBurst,
paused = !animationEnabled,
),
modifier = Modifier
.fillMaxSize()
@@ -2208,6 +2490,21 @@ fun ChatScreen(
LocalRelayServerImageResolver provides relayServerImageResolver,
LocalThinkingIndicator provides thinkingIndicatorConfig,
) {
val petVisitTargetUiKey = remember(messages) {
newestPetVisitTargetUiKey(messages)
}
val petPerchUiKey = remember(messages) {
newestPetPerchUiKey(messages)
}
val petJourneyPerchUiKeys = remember(messages) {
petPerchUiKeys(messages)
}
val visibleMessageKeys by remember(listState) {
derivedStateOf {
listState.layoutInfo.visibleItemsInfo
.mapTo(mutableSetOf<Any>()) { it.key }
}
}
LazyColumn(
state = listState,
modifier = Modifier
@@ -2293,6 +2590,32 @@ fun ChatScreen(
MessageBubble(
message = message,
modifier = bubbleModifier,
petVisitTargetKey = if (
message.uiKey == petVisitTargetUiKey &&
message.uiKey in visibleMessageKeys
) {
"chat-message:${message.uiKey}"
} else {
null
},
petPerchKey = if (
message.uiKey in petJourneyPerchUiKeys &&
message.uiKey in visibleMessageKeys
) {
val prefix = if (message.role == MessageRole.USER) {
CHAT_PET_USER_MESSAGE_PERCH_PREFIX
} else {
CHAT_PET_ASSISTANT_MESSAGE_PERCH_PREFIX
}
val marker = if (message.uiKey == petPerchUiKey) {
""
} else {
CHAT_PET_STEP_MESSAGE_MARKER
}
"$prefix$marker${message.uiKey}"
} else {
null
},
maxBubbleWidth = maxBubbleWidth,
showThinking = showThinking,
isFirstInGroup = isFirstInGroup,
@@ -2308,24 +2631,8 @@ fun ChatScreen(
onAttachmentManualFetch = { msgId, idx ->
chatViewModel.manualFetchAttachment(msgId, idx)
},
onCardAction = { msgId, cardKey, action ->
// OPEN_URL is resolved at the UI layer
// because launching ACTION_VIEW needs a
// Context. Record the dispatch first so
// the card collapses even if launch fails.
if (action.mode == com.hermesandroid.relay.data.HermesCardAction.Modes.OPEN_URL) {
chatViewModel.dispatchCardAction(msgId, cardKey, action)
com.hermesandroid.relay.ui.components.handleCardActionExternally(
context,
action,
)
} else {
chatViewModel.dispatchCardAction(msgId, cardKey, action)
}
},
onCardInput = { msgId, cardKey, value ->
chatViewModel.answerAsk(msgId, cardKey, value)
},
onCardAction = handleCardAction,
onCardInput = handleCardInput,
onEditMessage = if (
isGatewayTransport &&
!isStreaming &&
@@ -2352,6 +2659,14 @@ fun ChatScreen(
"$inputText\n$quoted\n\n"
}
},
onSpeakMessage = if (
conversationVoiceResponseActionsEnabled &&
!isStreaming
) {
{ text -> voiceViewModel.speakResponse(text) }
} else {
null
},
onCopyMessage = { text ->
haptic.performHapticFeedback(HapticFeedbackType.LongPress)
// The new Clipboard API is suspend-based, so the
@@ -2460,9 +2775,31 @@ fun ChatScreen(
.padding(16.dp)
.zIndex(8f)
) {
Box(
modifier = Modifier
// The visible FAB is narrower than the pet's
// footprint. Measure a transparent landing
// ledge around it so the pet can stand above
// the control without covering its touch area.
.width(72.dp)
.height(48.dp)
.petPerchSurface(
key = CHAT_SCROLL_TO_BOTTOM_PET_PERCH,
routes = CHAT_PET_ROUTES,
),
contentAlignment = Alignment.Center,
) {
SmallFloatingActionButton(
modifier = Modifier
.size(48.dp)
// The surrounding box is a landing ledge, but
// the real control remains forbidden space so
// composer and bubble routes cannot pass the
// pet's complete scaled footprint over it.
.petObstacleSurface(
key = CHAT_SCROLL_TO_BOTTOM_PET_OBSTACLE,
routes = CHAT_PET_ROUTES,
)
.semantics {
contentDescription = if (unreadMessageCount > 0) {
"Scroll to bottom, $unreadMessageCount unread " +
@@ -2503,6 +2840,7 @@ fun ChatScreen(
)
}
}
}
}
// Copy feedback snackbar
@@ -2763,8 +3101,8 @@ fun ChatScreen(
}
val inputCaption = when {
isStreaming && hasContent && steerableTurn ->
"↳ sends now — Hermes adjusts mid-turn"
isStreaming && hasContent -> "↳ delivered after this turn finishes"
stringResource(R.string.chat_sends_now)
isStreaming && hasContent -> stringResource(R.string.chat_delivered_after_turn)
isStreaming && steerNotice != null -> steerNotice
else -> null
}
@@ -2801,7 +3139,7 @@ fun ChatScreen(
val serverDefaultModelDetail = AgentDisplay.displayModelName(serverModelName)
?: AgentDisplay.displayModelName(effectiveProfile?.model)
val hasModelChoices = modelProviders.any { it.models.isNotEmpty() } || sseModelOptions.isNotEmpty()
val serverDefaultLabel = stringResource(R.string.chat_server_default_sessions)
val serverDefaultLabel = stringResource(R.string.chat_server_default)
val notOnPlanLabel = stringResource(R.string.chat_not_on_plan)
val needsSetupLabel = stringResource(R.string.chat_needs_setup)
val modelDefaultLabel = stringResource(R.string.chat_model_label)
@@ -2926,6 +3264,7 @@ fun ChatScreen(
} else {
null
}
ChatInputBar(
value = inputText,
onValueChange = { inputText = it },
@@ -2979,7 +3318,7 @@ fun ChatScreen(
com.hermesandroid.relay.viewmodel.StandardVoiceAvailability.Unsupported ->
"This Hermes build has no voice routes — update hermes-agent or pair Relay"
else ->
"Voice needs a reachable Hermes dashboard or Relay voice route"
context.getString(R.string.chat_voice_needs_route)
},
Toast.LENGTH_SHORT,
).show()
@@ -3013,7 +3352,7 @@ fun ChatScreen(
charLimit = charLimit,
caption = turnStatus ?: inputCaption,
voiceReady = voiceReady,
showVoiceHint = !voiceHintSeen,
showVoiceHint = !voiceHintSeen && !conversationVoiceDockVisible,
onVoiceHintShown = { connectionViewModel.setVoiceHintSeen(true) },
isDarkTheme = isDarkTheme,
modelControl = modelControl,
@@ -3028,6 +3367,37 @@ fun ChatScreen(
onEffortOptionSelected = { option ->
option.value?.let { chatViewModel.selectReasoningEffort(it) }
},
topContent = {
ConversationVoiceDock(
uiState = voiceUiState,
engineMode = voiceStats.voiceEngineMode,
provider = activeVoiceProvider,
model = activeVoiceModel,
voice = activeVoiceName,
profileName = AgentDisplay.profileDisplayName(effectiveProfile),
outputEnabled = activeVoiceEnabled,
onMicTap = { voiceViewModel.startListening() },
onMicRelease = { voiceViewModel.stopListening() },
onInterrupt = { voiceViewModel.interruptSpeaking() },
onPauseAutoMode = { voiceViewModel.pauseContinuousMode() },
onModeChange = { voiceViewModel.setInteractionMode(it) },
onFocusRequest = {
setVoicePresentationMode(VoicePresentationMode.Focus)
},
onOverlayRequest = showVoiceSystemOverlay,
onOpenSettings = onNavigateToVoiceSettings,
onExit = { voiceViewModel.exitVoiceMode() },
)
},
topContentVisible = conversationVoiceDockVisible,
suppressVoiceTrailing = conversationVoiceDockVisible,
// Measure the existing composer as a Desktop-style ledge. The
// floating host stands on its top edge; no transcript space is
// reserved and the composer controls remain unobstructed.
modifier = Modifier.petPerchSurface(
key = CHAT_PET_WALK_REGION,
routes = CHAT_PET_ROUTES,
),
enabled = chatReady,
onModelPickerClick = { showModelSheet = true },
)
@@ -3181,17 +3551,15 @@ fun ChatScreen(
voiceOutputModel = activeVoiceModel,
voiceOutputVoice = activeVoiceName,
voiceProfileName = AgentDisplay.profileDisplayName(effectiveProfile),
voiceConfigScope = activeVoiceScope,
voiceOutputEnabled = activeVoiceEnabled,
voiceOutputFallbackEnabled = voiceOutputConfig?.fallback_enabled,
presentationMode = effectiveVoicePresentationMode,
onPresentationModeChange = setVoicePresentationMode,
onOverlayRequest = showVoiceSystemOverlay,
// Gear button in the overlay's expanded controls. The overlay
// exits voice mode before invoking this, so navigation lands
// on Voice Settings with no overlay left on top.
onOpenSettings = onNavigateToVoiceSettings,
onCompactModeChange = { compact ->
voiceCompactMode = compact
},
// === v0.4.1 JIT permission-denied chip ===
// Tap deep-links to Settings → Apps → Hermes-Relay →
// Permissions for the running package. Use BuildConfig
@@ -3215,6 +3583,8 @@ fun ChatScreen(
onHermesConfirmationAnswer = { answer ->
voiceViewModel.answerHermesConfirmation(answer)
},
onCardAction = handleCardAction,
onCardInput = handleCardInput,
// === END v0.4.1 ===
)
}
@@ -3326,6 +3696,7 @@ private fun ChatColdStartLoadingState(
onNavigateToConnections: () -> Unit,
modifier: Modifier = Modifier,
) {
val context = LocalContext.current
val commands = remember(
connectionLabel,
chatMode,
@@ -3335,6 +3706,7 @@ private fun ChatColdStartLoadingState(
isLoadingSessions,
) {
buildChatLoadingCommands(
context = context,
connectionLabel = connectionLabel,
chatMode = chatMode,
apiReachable = apiReachable,
@@ -3348,12 +3720,13 @@ private fun ChatColdStartLoadingState(
modifier = modifier,
contentAlignment = Alignment.Center,
) {
if (animationEnabled) {
if (LocalBackgroundVisualizationEnabled.current) {
LocalAgentAvatar.current.Render(
state = AvatarRenderState(
state = SphereState.Thinking,
intensity = streamingIntensity.coerceAtLeast(0.18f),
toolCallBurst = toolCallBurst,
paused = !animationEnabled,
),
modifier = Modifier
.fillMaxSize()
@@ -3434,6 +3807,7 @@ private fun ChatColdStartLoadingState(
}
private fun buildChatLoadingCommands(
context: android.content.Context,
connectionLabel: String?,
chatMode: ChatMode,
apiReachable: Boolean,
@@ -3443,15 +3817,15 @@ private fun buildChatLoadingCommands(
): List<ChatLoadingCommand> {
val hasConnection = !connectionLabel.isNullOrBlank()
val chatModeDetail = when (chatMode) {
ChatMode.ENHANCED_HERMES -> "sessions stream"
ChatMode.PORTABLE -> "portable stream"
ChatMode.ENHANCED_HERMES -> context.getString(R.string.chat_stream_sessions)
ChatMode.PORTABLE -> context.getString(R.string.chat_stream_portable)
ChatMode.DISCONNECTED -> "waiting"
}
return listOf(
ChatLoadingCommand(
state = if (hasConnection) ChatLoadingCommandState.Done else ChatLoadingCommandState.Active,
command = "/state restore",
detail = if (hasConnection) "active config loaded" else "loading config",
detail = if (hasConnection) context.getString(R.string.chat_config_active) else context.getString(R.string.chat_config_loading),
),
ChatLoadingCommand(
state = when {
@@ -3459,7 +3833,7 @@ private fun buildChatLoadingCommands(
else -> ChatLoadingCommandState.Pending
},
command = "/route resolve",
detail = connectionLabel?.takeIf { it.isNotBlank() } ?: "selecting route",
detail = connectionLabel?.takeIf { it.isNotBlank() } ?: context.getString(R.string.chat_selecting_route),
),
ChatLoadingCommand(
state = when {
@@ -3468,7 +3842,7 @@ private fun buildChatLoadingCommands(
else -> ChatLoadingCommandState.Pending
},
command = "/hermes ping",
detail = if (apiReachable) "online" else "contacting server",
detail = if (apiReachable) "online" else context.getString(R.string.chat_contacting_server),
),
ChatLoadingCommand(
state = when {
@@ -3713,7 +4087,7 @@ private fun ingestAttachmentFromUri(
)
)
} catch (e: Exception) {
Toast.makeText(context, "Failed to read file", Toast.LENGTH_SHORT).show()
Toast.makeText(context, context.getString(R.string.chat_failed_read_file), Toast.LENGTH_SHORT).show()
}
}
@@ -1,18 +1,19 @@
package com.hermesandroid.relay.ui.screens
import android.webkit.CookieManager
import android.webkit.WebChromeClient
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebView
import android.webkit.WebViewClient
import androidx.activity.compose.BackHandler
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.padding
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.filled.ArrowBack
import androidx.compose.material.icons.filled.Close
import androidx.compose.material.icons.filled.Check
import androidx.compose.material3.Button
import androidx.compose.material3.Card
@@ -21,6 +22,7 @@ import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Scaffold
@@ -38,11 +40,11 @@ import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.Alignment
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.platform.LocalResources
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.input.PasswordVisualTransformation
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.compose.ui.window.Dialog
import com.hermesandroid.relay.R
import com.hermesandroid.relay.ui.components.ConnectionSetupTimeline
import com.hermesandroid.relay.ui.components.ConnectionSetupTimelineStep
@@ -50,10 +52,10 @@ import com.hermesandroid.relay.network.upstream.DashboardApiClient
import com.hermesandroid.relay.network.upstream.DashboardAuthProvider
import com.hermesandroid.relay.network.upstream.DashboardAuthSession
import com.hermesandroid.relay.network.upstream.DashboardCookieStore
import com.hermesandroid.relay.network.upstream.EncryptedDashboardCookieStore
import com.hermesandroid.relay.network.upstream.DashboardRedirectAuthMode
import com.hermesandroid.relay.network.upstream.EncryptedDashboardCookieStore
import com.hermesandroid.relay.network.upstream.NativeDashboardSignInCoordinator
import com.hermesandroid.relay.network.upstream.dashboardRedirectAuthMode
import com.hermesandroid.relay.network.upstream.androidDashboardRedirectAuthMode
import com.hermesandroid.relay.network.upstream.importDashboardCookieHeader
import com.hermesandroid.relay.network.upstream.isNativeDashboardTransportEligible
import com.hermesandroid.relay.viewmodel.ConnectionViewModel
@@ -62,6 +64,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
/**
* Connection-level Dashboard authentication flow. It is deliberately outside
@@ -76,6 +79,7 @@ fun DashboardSignInScreen(
onAuthenticated: () -> Unit,
) {
val context = LocalContext.current
val resources = LocalResources.current
val appContext = context.applicationContext
val scope = rememberCoroutineScope()
val activeConnection by connectionViewModel.activeConnection.collectAsState()
@@ -90,8 +94,8 @@ fun DashboardSignInScreen(
var actionMessage by remember { mutableStateOf<String?>(null) }
var actionIsError by remember { mutableStateOf(false) }
var oauthProvider by remember { mutableStateOf<DashboardAuthProvider?>(null) }
var redirectAuthMode by remember(dashboardUrl, connectionId) {
mutableStateOf(DashboardRedirectAuthMode.WebView)
var authFlows by remember(dashboardUrl, connectionId) {
mutableStateOf<List<String>>(emptyList())
}
var nativeSignInJob by remember(dashboardUrl, connectionId) { mutableStateOf<Job?>(null) }
var authenticationComplete by remember { mutableStateOf(false) }
@@ -135,20 +139,20 @@ fun DashboardSignInScreen(
LaunchedEffect(dashboardUrl, connectionId) {
if (dashboardUrl.isBlank()) {
loading = false
actionMessage = context.getString(R.string.dashboard_no_url_configured)
actionMessage = resources.getString(R.string.dashboard_no_url_configured)
return@LaunchedEffect
}
val client = clientFactory()
try {
val status = client.getStatus().getOrElse {
actionMessage = it.message ?: context.getString(R.string.dashboard_request_failed)
actionMessage = it.message ?: resources.getString(R.string.dashboard_request_failed)
actionIsError = true
return@LaunchedEffect
}
providers = client.getAuthProviders().getOrNull()
?.takeIf { it.isNotEmpty() }
?: status.authProviderDetails
redirectAuthMode = dashboardRedirectAuthMode(status.authFlows)
authFlows = status.authFlows
val session = if (status.authRequired) client.currentSession().getOrNull() else null
connectionViewModel.recordDashboardStatus(
status = status,
@@ -179,11 +183,11 @@ fun DashboardSignInScreen(
finishAuthentication()
} else {
actionMessage = result.exceptionOrNull()?.message
?: context.getString(R.string.dashboard_signin_no_session)
?: resources.getString(R.string.dashboard_signin_no_session)
actionIsError = true
}
} catch (e: Exception) {
actionMessage = e.message ?: context.getString(R.string.dashboard_signin_failed)
actionMessage = e.message ?: resources.getString(R.string.dashboard_signin_failed)
actionIsError = true
} finally {
actionInFlight = false
@@ -194,25 +198,28 @@ fun DashboardSignInScreen(
fun startRedirectSignIn(provider: DashboardAuthProvider) {
if (actionInFlight || dashboardUrl.isBlank()) return
if (redirectAuthMode == DashboardRedirectAuthMode.WebView) {
if (
androidDashboardRedirectAuthMode(provider.name, authFlows) ==
DashboardRedirectAuthMode.WebView
) {
oauthProvider = provider
return
}
if (!isNativeDashboardTransportEligible(dashboardUrl)) {
actionMessage = context.getString(R.string.dashboard_native_signin_requires_https)
actionMessage = resources.getString(R.string.dashboard_native_signin_requires_https)
actionIsError = true
return
}
val authClient = connectionViewModel.nativeDashboardAuthClientForActive(dashboardUrl)
if (authClient == null) {
actionMessage = context.getString(R.string.dashboard_native_signin_unavailable)
actionMessage = resources.getString(R.string.dashboard_native_signin_unavailable)
actionIsError = true
return
}
actionInFlight = true
actionIsError = false
actionMessage = context.getString(R.string.dashboard_native_signin_opening)
actionMessage = resources.getString(R.string.dashboard_native_signin_opening)
nativeSignInJob = scope.launch {
try {
NativeDashboardSignInCoordinator(authClient).signIn(provider.name) { authorizationUrl ->
@@ -228,19 +235,19 @@ fun DashboardSignInScreen(
}
if (session?.authenticated == true) {
actionMessage = session.provider?.let {
context.getString(R.string.dashboard_signed_in_with, it)
} ?: context.getString(R.string.dashboard_signed_in)
resources.getString(R.string.dashboard_signed_in_with, it)
} ?: resources.getString(R.string.dashboard_signed_in)
actionIsError = false
finishAuthentication()
} else {
actionMessage = context.getString(R.string.dashboard_signin_no_session)
actionMessage = resources.getString(R.string.dashboard_signin_no_session)
actionIsError = true
}
} catch (cancelled: CancellationException) {
throw cancelled
} catch (error: Exception) {
actionMessage = error.message
?: context.getString(R.string.dashboard_signin_failed)
?: resources.getString(R.string.dashboard_signin_failed)
actionIsError = true
} finally {
actionInFlight = false
@@ -253,10 +260,8 @@ fun DashboardSignInScreen(
onDispose { nativeSignInJob?.cancel() }
}
oauthProvider
?.takeIf { redirectAuthMode == DashboardRedirectAuthMode.WebView }
?.let { provider ->
DashboardOAuthDialog(
oauthProvider?.let { provider ->
DashboardOAuthScreen(
dashboardUrl = dashboardUrl,
provider = provider,
cookieStoreFactory = cookieStoreFactory,
@@ -272,8 +277,8 @@ fun DashboardSignInScreen(
client.shutdown()
}
actionMessage = session.provider?.let {
context.getString(R.string.dashboard_signed_in_with, it)
} ?: context.getString(R.string.dashboard_signed_in)
resources.getString(R.string.dashboard_signed_in_with, it)
} ?: resources.getString(R.string.dashboard_signed_in)
finishAuthentication()
}
},
@@ -282,6 +287,7 @@ fun DashboardSignInScreen(
actionIsError = true
},
)
return
}
Scaffold(
@@ -289,10 +295,7 @@ fun DashboardSignInScreen(
TopAppBar(
title = { Text(stringResource(R.string.dashboard_sign_in)) },
navigationIcon = {
IconButton(onClick = {
nativeSignInJob?.cancel()
onBack()
}) {
IconButton(onClick = onBack) {
Icon(
Icons.AutoMirrored.Filled.ArrowBack,
contentDescription = stringResource(R.string.dashboard_back),
@@ -321,13 +324,11 @@ fun DashboardSignInScreen(
actionInFlight = actionInFlight,
actionMessage = actionMessage,
actionIsError = actionIsError,
nativePkce = redirectAuthMode == DashboardRedirectAuthMode.NativePkce,
nativeSignInInFlight = nativeSignInJob != null,
nativeTransportEligible = isNativeDashboardTransportEligible(dashboardUrl),
onSignIn = ::submitPassword,
onOAuthSignIn = ::startRedirectSignIn,
onCancelNativeSignIn = {
actionMessage = context.getString(R.string.dashboard_native_signin_cancelled)
actionMessage = resources.getString(R.string.dashboard_native_signin_cancelled)
actionIsError = false
nativeSignInJob?.cancel()
},
@@ -396,9 +397,7 @@ private fun DashboardSignInForm(
actionInFlight: Boolean,
actionMessage: String?,
actionIsError: Boolean,
nativePkce: Boolean,
nativeSignInInFlight: Boolean,
nativeTransportEligible: Boolean,
onSignIn: (String, String, String) -> Unit,
onOAuthSignIn: (DashboardAuthProvider) -> Unit,
onCancelNativeSignIn: () -> Unit,
@@ -427,18 +426,19 @@ private fun DashboardSignInForm(
redirectProviders.forEach { provider ->
Button(
onClick = { onOAuthSignIn(provider) },
enabled = !actionInFlight && (!nativePkce || nativeTransportEligible),
enabled = !actionInFlight,
modifier = Modifier.fillMaxWidth(),
) {
Text(stringResource(R.string.dashboard_signin_with_provider, provider.displayName ?: provider.name))
}
}
if (nativePkce && !nativeTransportEligible && redirectProviders.isNotEmpty()) {
Text(
text = stringResource(R.string.dashboard_native_signin_requires_https),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.error,
)
if (nativeSignInInFlight) {
Button(
onClick = onCancelNativeSignIn,
modifier = Modifier.fillMaxWidth(),
) {
Text(stringResource(R.string.dashboard_cancel))
}
}
if (passwordProvider != null || providers.isEmpty()) {
if (redirectProviders.isNotEmpty()) HorizontalDivider()
@@ -476,18 +476,11 @@ private fun DashboardSignInForm(
},
)
}
if (nativeSignInInFlight) {
Button(
onClick = onCancelNativeSignIn,
modifier = Modifier.fillMaxWidth(),
) {
Text(stringResource(R.string.dashboard_cancel))
}
}
}
@OptIn(ExperimentalMaterial3Api::class)
@Composable
private fun DashboardOAuthDialog(
private fun DashboardOAuthScreen(
dashboardUrl: String,
provider: DashboardAuthProvider,
cookieStoreFactory: () -> DashboardCookieStore,
@@ -504,6 +497,8 @@ private fun DashboardOAuthDialog(
val verifyFailedStatus = stringResource(R.string.dashboard_oauth_verify_failed)
var statusText by remember(initialStatus) { mutableStateOf(initialStatus) }
var checking by remember { mutableStateOf(false) }
var pageProgress by remember { mutableStateOf(0) }
var webView by remember { mutableStateOf<WebView?>(null) }
val loginUrl = remember(dashboardUrl, provider.name) {
DashboardApiClient.authLoginUrl(
baseUrl = dashboardUrl,
@@ -512,14 +507,17 @@ private fun DashboardOAuthDialog(
)
}
fun maybeVerify(url: String?) {
fun handleNavigation(url: String?) {
val loadedUrl = url?.takeIf { it.isNotBlank() } ?: return
val root = dashboardUrl.trim().trimEnd('/')
val relative = loadedUrl.trim().removePrefix(root)
val stillAuthenticating = relative.startsWith("/login", true) ||
relative.startsWith("/auth/login", true) ||
relative.startsWith("/auth/callback", true)
if (!loadedUrl.startsWith(root, true) || stillAuthenticating) return
when (dashboardWebViewAuthNavigation(dashboardUrl, loadedUrl)) {
DashboardWebViewAuthNavigation.Continue -> return
DashboardWebViewAuthNavigation.RejectLoopbackCallback -> {
statusText = notAcceptedStatus
onError(notAcceptedStatus)
return
}
DashboardWebViewAuthNavigation.ImportAndVerify -> Unit
}
val manager = CookieManager.getInstance()
manager.flush()
val imported = importDashboardCookieHeader(
@@ -549,39 +547,162 @@ private fun DashboardOAuthDialog(
}
}
Dialog(onDismissRequest = onDismiss) {
Card(modifier = Modifier.fillMaxWidth().heightIn(max = 640.dp)) {
Column(
modifier = Modifier.padding(12.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
IconButton(onClick = onDismiss) {
Icon(Icons.Filled.Close, contentDescription = stringResource(R.string.dashboard_close_signin))
}
Text(statusText, style = MaterialTheme.typography.bodySmall)
AndroidView(
modifier = Modifier.fillMaxWidth().weight(1f),
factory = { viewContext ->
CookieManager.getInstance().setAcceptCookie(true)
WebView(viewContext).apply {
settings.javaScriptEnabled = true
settings.domStorageEnabled = true
webViewClient = object : WebViewClient() {
override fun shouldOverrideUrlLoading(
view: WebView,
request: WebResourceRequest,
): Boolean = false
BackHandler(onBack = onDismiss)
override fun onPageFinished(view: WebView, url: String?) {
super.onPageFinished(view, url)
maybeVerify(url)
}
}
loadUrl(loginUrl)
}
},
DisposableEffect(Unit) {
onDispose {
webView?.stopLoading()
webView?.destroy()
webView = null
}
}
Scaffold(
topBar = {
TopAppBar(
title = {
Column {
Text(
text = stringResource(
R.string.dashboard_signin_with_provider,
provider.displayName ?: provider.name,
),
style = MaterialTheme.typography.titleMedium,
)
Text(
text = statusText,
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
)
}
},
navigationIcon = {
IconButton(onClick = onDismiss) {
Icon(
Icons.AutoMirrored.Filled.ArrowBack,
contentDescription = stringResource(R.string.dashboard_back),
)
}
},
)
},
) { innerPadding ->
Column(
modifier = Modifier
.fillMaxSize()
.padding(innerPadding),
) {
if (pageProgress in 0..99) {
LinearProgressIndicator(
progress = { pageProgress / 100f },
modifier = Modifier.fillMaxWidth(),
)
}
AndroidView(
modifier = Modifier
.fillMaxWidth()
.weight(1f),
factory = { viewContext ->
CookieManager.getInstance().setAcceptCookie(true)
WebView(viewContext).apply {
settings.javaScriptEnabled = true
settings.domStorageEnabled = true
webChromeClient = object : WebChromeClient() {
override fun onProgressChanged(view: WebView, newProgress: Int) {
pageProgress = newProgress
}
}
webViewClient = object : WebViewClient() {
override fun shouldOverrideUrlLoading(
view: WebView,
request: WebResourceRequest,
): Boolean {
val target = request.url.toString()
if (
dashboardWebViewAuthNavigation(dashboardUrl, target) ==
DashboardWebViewAuthNavigation.RejectLoopbackCallback
) {
handleNavigation(target)
return true
}
return false
}
override fun onReceivedError(
view: WebView,
request: WebResourceRequest,
error: WebResourceError,
) {
super.onReceivedError(view, request, error)
if (request.isForMainFrame) {
val message = error.description?.toString()
?.takeIf { it.isNotBlank() }
?: verifyFailedStatus
statusText = message
onError(message)
}
}
override fun onPageFinished(view: WebView, url: String?) {
super.onPageFinished(view, url)
handleNavigation(url)
}
}
webView = this
loadUrl(loginUrl)
}
},
)
}
}
}
internal enum class DashboardWebViewAuthNavigation {
Continue,
ImportAndVerify,
RejectLoopbackCallback,
}
/**
* Android redirect providers use the dashboard's cookie/OIDC flow. A foreign
* loopback callback belongs to the desktop native-PKCE contract and must never
* be followed, imported, or treated as an authenticated Android return.
*/
internal fun dashboardWebViewAuthNavigation(
dashboardUrl: String,
loadedUrl: String,
): DashboardWebViewAuthNavigation {
val dashboard = dashboardUrl.trim().trimEnd('/').toHttpUrlOrNull()
?: return DashboardWebViewAuthNavigation.Continue
val loaded = loadedUrl.trim().toHttpUrlOrNull()
?: return DashboardWebViewAuthNavigation.Continue
val sameOrigin = dashboard.scheme == loaded.scheme &&
dashboard.host.equals(loaded.host, ignoreCase = true) &&
dashboard.port == loaded.port
if (!sameOrigin) {
val foreignLoopback = loaded.scheme == "http" &&
loaded.host in setOf("127.0.0.1", "localhost", "::1") &&
loaded.encodedPath == "/callback"
return if (foreignLoopback) {
DashboardWebViewAuthNavigation.RejectLoopbackCallback
} else {
DashboardWebViewAuthNavigation.Continue
}
}
val basePath = dashboard.encodedPath.trimEnd('/')
val relativePath = loaded.encodedPath
.removePrefix(basePath)
.ifBlank { "/" }
return if (
relativePath.equals("/login", ignoreCase = true) ||
relativePath.equals("/auth/login", ignoreCase = true)
) {
DashboardWebViewAuthNavigation.Continue
} else {
// Includes the public /auth/callback response: import its cookies at
// root scope, then verify the resulting session through /api/auth/me.
DashboardWebViewAuthNavigation.ImportAndVerify
}
}
@@ -34,11 +34,13 @@ import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Scaffold
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.material3.TopAppBar
import androidx.compose.material3.TopAppBarDefaults
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
@@ -75,6 +77,7 @@ fun DeveloperSettingsScreen(
val context = LocalContext.current
val scope = rememberCoroutineScope()
val isDarkTheme = LocalBrand.current.isDark
val petTerrainOverlayEnabled by FeatureFlags.petTerrainOverlayEnabled(context).collectAsState(false)
// Data management local state — unfolded from the private
// DataManagementSection helper in the old SettingsScreen.
@@ -380,6 +383,34 @@ fun DeveloperSettingsScreen(
}
HorizontalDivider()
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = Alignment.CenterVertically,
) {
Column(modifier = Modifier.weight(1f)) {
Text(
text = stringResource(R.string.dev_settings_pet_terrain_overlay),
style = MaterialTheme.typography.bodyMedium,
)
Text(
text = stringResource(R.string.dev_settings_pet_terrain_overlay_desc),
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
Switch(
checked = petTerrainOverlayEnabled,
onCheckedChange = { enabled ->
scope.launch {
FeatureFlags.setPetTerrainOverlayEnabled(context, enabled)
}
},
)
}
HorizontalDivider()
}
// Lock developer options
@@ -0,0 +1,32 @@
package com.hermesandroid.relay.ui.screens
import androidx.annotation.StringRes
import com.hermesandroid.relay.R
import com.hermesandroid.relay.data.PetTemperament
internal data class PetTemperamentOption(
val temperament: PetTemperament,
@StringRes val labelRes: Int,
@StringRes val descriptionRes: Int,
)
internal val petTemperamentOptions: List<PetTemperamentOption> = listOf(
PetTemperamentOption(
temperament = PetTemperament.Calm,
labelRes = R.string.appearance_pet_temperament_calm,
descriptionRes = R.string.appearance_pet_temperament_calm_desc,
),
PetTemperamentOption(
temperament = PetTemperament.Balanced,
labelRes = R.string.appearance_pet_temperament_balanced,
descriptionRes = R.string.appearance_pet_temperament_balanced_desc,
),
PetTemperamentOption(
temperament = PetTemperament.Playful,
labelRes = R.string.appearance_pet_temperament_playful,
descriptionRes = R.string.appearance_pet_temperament_playful_desc,
),
)
internal fun petTemperamentOption(temperament: PetTemperament): PetTemperamentOption =
petTemperamentOptions.first { it.temperament == temperament }

Some files were not shown because too many files have changed in this diff Show More