Compare commits

..
Author SHA1 Message Date
Bailey Dixon 0213dbf5db fix(android): align wake native runtime 2026-08-28 21:22:28 -04:00
18 changed files with 533 additions and 353 deletions
+3 -1
View File
@@ -16,7 +16,9 @@ function classifyCiPaths(paths) {
android: forceAll || under(['app/', 'relay-core/', 'relay-ui/', 'ui-preview/', 'quest/', 'gradle/']) || exact([
'build.gradle.kts', 'settings.gradle.kts', 'gradle.properties', 'gradlew', 'gradlew.bat',
'scripts/check-android-locales.py', 'scripts/android-locale-harness.py',
'scripts/check-android-collection-apis.py', 'scripts/check-android-release-notes.py',
'scripts/check-android-collection-apis.py', 'scripts/check-android-native-compat.py',
'scripts/check-android-release-notes.py',
'scripts/tests/check_android_native_compat_test.py',
'scripts/tests/check_android_release_notes_test.py', '.github/workflows/ci-android.yml',
'.github/workflows/play-preflight-android.yml',
'.github/workflows/approve-release-android.yml',
@@ -16,6 +16,8 @@ assert.deepEqual(classifyCiPaths(['README.md']), none);
assert.deepEqual(classifyCiPaths(['desktop/src/cli.ts']), { ...none, desktop: true });
assert.deepEqual(classifyCiPaths(['relay-core/src/main/kotlin/Wire.kt']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['scripts/check-android-release-notes.py']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['scripts/check-android-native-compat.py']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['scripts/tests/check_android_native_compat_test.py']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['plugin/relay/server.py']), { ...none, plugin: true });
assert.deepEqual(classifyCiPaths(['plugin/dashboard/src/App.tsx']), { ...none, dashboard: true });
assert.deepEqual(classifyCiPaths(['user-docs/index.md']), { ...none, docs: true });
+17
View File
@@ -33,7 +33,9 @@ on:
- "scripts/check-android-locales.py"
- "scripts/android-locale-harness.py"
- "scripts/check-android-collection-apis.py"
- "scripts/check-android-native-compat.py"
- "scripts/check-android-release-notes.py"
- "scripts/tests/check_android_native_compat_test.py"
- "scripts/tests/check_android_release_notes_test.py"
- ".github/workflows/ci-android.yml"
- ".github/workflows/play-preflight-android.yml"
@@ -80,6 +82,9 @@ jobs:
python3 scripts/check-android-release-notes.py
python3 -m unittest scripts.tests.check_android_release_notes_test
- name: Test Android native compatibility checker
run: python3 -m unittest scripts.tests.check_android_native_compat_test
- name: Run Android lint
run: ./gradlew lint --console=plain
@@ -109,6 +114,12 @@ jobs:
- name: Build debug APK
run: ./gradlew assembleDebug --console=plain
- name: Verify packaged ONNX Runtime compatibility
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/googlePlay/debug/*.apk \
app/build/outputs/apk/sideload/debug/*.apk
- name: Upload debug APK
uses: actions/upload-artifact@v7
if: ${{ github.ref == 'refs/heads/main' }}
@@ -225,3 +236,9 @@ jobs:
python3 scripts/check-android-collection-apis.py \
--apk app/build/outputs/apk/googlePlay/release/*.apk \
--apk app/build/outputs/apk/sideload/release/*.apk
- name: Verify packaged ONNX Runtime compatibility
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/googlePlay/release/*.apk \
app/build/outputs/apk/sideload/release/*.apk
+13
View File
@@ -263,6 +263,19 @@ jobs:
python3 scripts/check-android-collection-apis.py \
--apk app/build/outputs/apk/sideload/candidate/*.apk
- name: Verify stable packaged ONNX Runtime compatibility
if: ${{ needs.validate.outputs.prerelease != 'true' }}
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/googlePlay/release/*.apk \
app/build/outputs/apk/sideload/release/*.apk
- name: Verify candidate packaged ONNX Runtime compatibility
if: ${{ needs.validate.outputs.prerelease == 'true' }}
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/sideload/candidate/*.apk
- name: List produced artifacts (debug aid)
run: |
echo "=== APK outputs ==="
+1 -1
View File
@@ -17,8 +17,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
### Fixed
- **Android wake-word detection now loads a compatible native ONNX Runtime.** Packaged sherpa and Java JNI consumers are checked against the shared runtime for every supported ABI before release.
- **The visible Android Sphere keeps its smooth procedural motion across startup and chat.** Backgrounded and motion-disabled surfaces remain still without reducing foreground animation to a stepped ambient pulse.
- **Android Assistant sessions explain when no speech was captured instead of appearing stuck at Ready.** Retry feedback survives the separate system overlay process, recreated session UI requests the current turn state, and locked sessions keep transcript, response, and technical error text private.
### Removed
+1 -6
View File
@@ -1300,12 +1300,7 @@ and whether the agent is waiting on the user.
permissions; exercise compact, expanded, collapsed, and full-Voice handoff
states, background tap-through, rotation and insets, cancel/back, microphone
denial, network failure, process kill/recreation, and wake→voice→wake
resumption. For background and keyguard capture, record `AudioRecord`, AppOps,
and foreground-service state: the user-installed app owns capture outside the
separate session process, so confirm whether the selected Assistant role is
sufficient on each target OS or whether activation needs an explicit,
activation-scoped microphone foreground-service lease. Measure idle battery
drain because third-party assistants do not
resumption. Measure idle battery drain because third-party assistants do not
receive Google's dedicated low-power hotword hardware.
- **Audio quality guardrails** — normalize output volume across realtime and
@@ -39,38 +39,14 @@ enum class AssistantSessionPhase {
Closed,
}
enum class AssistantSessionNotice {
NoSpeech,
}
data class AssistantSessionSnapshot(
val phase: AssistantSessionPhase = AssistantSessionPhase.Launching,
val transcript: String? = null,
val response: String = "",
val notice: AssistantSessionNotice? = null,
val error: String? = null,
val screenContextSupported: Boolean = false,
)
internal fun assistantSnapshotForPresentation(
snapshot: AssistantSessionSnapshot,
locked: Boolean,
): AssistantSessionSnapshot = if (locked) {
snapshot.copy(
transcript = null,
response = "",
error = null,
screenContextSupported = false,
)
} else {
snapshot
}
internal fun assistantSnapshotMatchesActivation(
expectedActivationId: String?,
receivedActivationId: String?,
): Boolean = expectedActivationId != null && expectedActivationId == receivedActivationId
object AssistantRole {
fun status(context: Context): AssistantRoleStatus {
val component = ComponentName(context, HermesVoiceInteractionService::class.java)
@@ -233,7 +209,6 @@ object AssistantSessionProtocol {
onFailure = { failure ->
publish(
application,
activation.id,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
@@ -244,19 +219,13 @@ object AssistantSessionProtocol {
return true
}
fun publish(
context: Context,
activationId: String,
snapshot: AssistantSessionSnapshot,
) {
fun publish(context: Context, snapshot: AssistantSessionSnapshot) {
context.sendBroadcast(
Intent(context, AssistantSessionStateReceiver::class.java).apply {
action = ACTION_STATUS
putExtra(EXTRA_ACTIVATION_ID, activationId)
putExtra(EXTRA_PHASE, snapshot.phase.name)
putExtra(EXTRA_TRANSCRIPT, snapshot.transcript)
putExtra(EXTRA_RESPONSE, snapshot.response)
putExtra(EXTRA_NOTICE, snapshot.notice?.name)
putExtra(EXTRA_ERROR, snapshot.error)
putExtra(EXTRA_SCREEN_CONTEXT_SUPPORTED, snapshot.screenContextSupported)
}
@@ -269,6 +238,10 @@ object AssistantSessionProtocol {
}
}
fun publish(context: Context, state: VoiceUiState) {
publish(context, snapshotFromVoiceState(state))
}
internal fun snapshotFromVoiceState(state: VoiceUiState): AssistantSessionSnapshot {
val phase = when {
!state.voiceMode -> AssistantSessionPhase.Closed
@@ -283,10 +256,7 @@ object AssistantSessionProtocol {
phase = phase,
transcript = state.transcribedText?.take(MAX_SESSION_TEXT_CHARS),
response = state.responseText.take(MAX_SESSION_TEXT_CHARS),
notice = state.assistantNotice,
error = state.error
?.takeIf { phase == AssistantSessionPhase.Error }
?.take(MAX_SESSION_ERROR_CHARS),
error = state.error?.take(MAX_SESSION_ERROR_CHARS),
)
}
@@ -380,9 +350,6 @@ object AssistantSessionProtocol {
phase = phase,
transcript = intent.getStringExtra(EXTRA_TRANSCRIPT),
response = intent.getStringExtra(EXTRA_RESPONSE).orEmpty(),
notice = intent.getStringExtra(EXTRA_NOTICE)?.let { raw ->
runCatching { AssistantSessionNotice.valueOf(raw) }.getOrNull()
},
error = intent.getStringExtra(EXTRA_ERROR),
screenContextSupported = intent.getBooleanExtra(
EXTRA_SCREEN_CONTEXT_SUPPORTED,
@@ -393,33 +360,24 @@ object AssistantSessionProtocol {
private const val MAX_SESSION_TEXT_CHARS = 4_000
private const val MAX_SESSION_ERROR_CHARS = 1_000
private const val EXTRA_NOTICE = "notice"
}
object AssistantSessionState {
private val _snapshot = MutableStateFlow(AssistantSessionSnapshot())
val snapshot: StateFlow<AssistantSessionSnapshot> = _snapshot.asStateFlow()
@Volatile private var activationId: String? = null
internal fun update(receivedActivationId: String?, snapshot: AssistantSessionSnapshot) {
if (!assistantSnapshotMatchesActivation(activationId, receivedActivationId)) return
internal fun update(snapshot: AssistantSessionSnapshot) {
_snapshot.value = snapshot
}
internal fun reset(activationId: String) {
this.activationId = activationId
internal fun reset() {
_snapshot.value = AssistantSessionSnapshot()
}
}
class AssistantSessionStateReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
AssistantSessionState.update(
receivedActivationId = intent.getStringExtra(
AssistantSessionProtocol.EXTRA_ACTIVATION_ID
),
snapshot = AssistantSessionProtocol.readSnapshot(intent),
)
AssistantSessionState.update(AssistantSessionProtocol.readSnapshot(intent))
}
}
@@ -465,7 +423,6 @@ class AssistantSessionLifecycleReceiver : BroadcastReceiver() {
onFailure = { failure ->
AssistantSessionProtocol.publish(
application,
id,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
@@ -473,7 +430,6 @@ class AssistantSessionLifecycleReceiver : BroadcastReceiver() {
)
},
)
application.runtime.republishAssistantSnapshot(id)
return
}
if (AssistantSessionProtocol.isStartAction(intent.action)) {
@@ -3,11 +3,6 @@ package com.hermesandroid.relay.assistant
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.drawable.ColorDrawable
import android.app.KeyguardManager
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.os.Bundle
import android.service.voice.VoiceInteractionSession
import android.service.voice.VoiceInteractionSessionService
@@ -72,7 +67,6 @@ import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.LifecycleRegistry
@@ -114,11 +108,6 @@ internal fun shouldCancelVoiceWhenSessionUiEnds(
presentation: AssistantSessionPresentation,
): Boolean = presentation == AssistantSessionPresentation.Overlay
internal fun assistantPresentationLocked(
currentKeyguardLocked: Boolean?,
fallbackLocked: Boolean,
): Boolean = currentKeyguardLocked ?: fallbackLocked
private class HermesVoiceInteractionSession(
private val service: HermesVoiceInteractionSessionService,
) : VoiceInteractionSession(service) {
@@ -129,19 +118,12 @@ private class HermesVoiceInteractionSession(
private var surfaceExpanded by mutableStateOf(false)
private var activationId: String? = null
private var manualMic = false
private var keyguardLocked by mutableStateOf(false)
private var expectScreenContext: Boolean? = null
private var pendingSemantic = AssistantSemanticContext()
private var pendingScreenshot: ByteArray? = null
private var screenContextUi by mutableStateOf(AssistantScreenContextUi())
private val contextStore = assistantContextStore(service)
private var heartbeatJob: Job? = null
private var keyguardReceiverRegistered = false
private val keyguardReceiver = object : BroadcastReceiver() {
override fun onReceive(context: Context?, intent: Intent?) {
refreshKeyguardState()
}
}
init {
scope.launch {
@@ -157,17 +139,6 @@ private class HermesVoiceInteractionSession(
override fun onCreate() {
super.onCreate()
ContextCompat.registerReceiver(
service,
keyguardReceiver,
IntentFilter().apply {
addAction(Intent.ACTION_SCREEN_OFF)
addAction(Intent.ACTION_SCREEN_ON)
addAction(Intent.ACTION_USER_PRESENT)
},
ContextCompat.RECEIVER_NOT_EXPORTED,
)
keyguardReceiverRegistered = true
window.window?.apply {
setBackgroundDrawable(ColorDrawable(android.graphics.Color.TRANSPARENT))
clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND)
@@ -184,7 +155,6 @@ private class HermesVoiceInteractionSession(
PersistedHermesRelayTheme {
AssistantSessionSurface(
expanded = surfaceExpanded,
locked = keyguardLocked,
screenContext = screenContextUi,
onExpandedChange = { surfaceExpanded = it },
onCancel = { finishSession(cancelVoice = true) },
@@ -213,22 +183,11 @@ private class HermesVoiceInteractionSession(
override fun onShow(args: Bundle?, showFlags: Int) {
super.onShow(args, showFlags)
refreshKeyguardState(
fallbackLocked = args?.getBoolean(
HermesVoiceInteractionService.EXTRA_FROM_KEYGUARD,
false,
) == true,
)
if (keyguardLocked) {
if (args?.getBoolean(HermesVoiceInteractionService.EXTRA_FROM_KEYGUARD, false) == true) {
window.window?.addFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
} else {
window.window?.clearFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
}
setUiEnabled(true)
val startsNewLifecycle = presentation == AssistantSessionPresentation.Inactive
@@ -236,10 +195,10 @@ private class HermesVoiceInteractionSession(
if (!startsNewLifecycle) return
surfaceExpanded = false
AssistantSessionState.reset()
screenContextUi = AssistantScreenContextUi()
activationId = args?.getString(AssistantSessionProtocol.EXTRA_ACTIVATION_ID)
?: UUID.randomUUID().toString()
AssistantSessionState.reset(activationId!!)
manualMic = args?.getBoolean(AssistantSessionProtocol.EXTRA_MANUAL_MIC, false) ?: false
expectScreenContext = args?.getBoolean(
AssistantSessionProtocol.EXTRA_EXPECT_SCREEN_CONTEXT,
@@ -341,10 +300,6 @@ private class HermesVoiceInteractionSession(
pendingSemantic = AssistantSemanticContext()
pendingScreenshot = null
screenContextUi = AssistantScreenContextUi()
if (keyguardReceiverRegistered) {
runCatching { service.unregisterReceiver(keyguardReceiver) }
keyguardReceiverRegistered = false
}
viewOwner.stop()
scope.cancel()
super.onDestroy()
@@ -364,7 +319,6 @@ private class HermesVoiceInteractionSession(
)
}.onFailure {
AssistantSessionState.update(
activationId,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open the voice session.",
@@ -383,7 +337,6 @@ private class HermesVoiceInteractionSession(
setUiEnabled(false)
}.onFailure {
AssistantSessionState.update(
activationId,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open full voice.",
@@ -424,14 +377,6 @@ private class HermesVoiceInteractionSession(
}
}
private fun refreshKeyguardState(fallbackLocked: Boolean = keyguardLocked) {
keyguardLocked = assistantPresentationLocked(
currentKeyguardLocked = service.getSystemService(KeyguardManager::class.java)
?.isKeyguardLocked,
fallbackLocked = fallbackLocked,
)
}
@RequiresApi(android.os.Build.VERSION_CODES.Q)
private fun stageAssistState(state: AssistState) {
stageAssistData(state.assistStructure, state.assistContent)
@@ -518,7 +463,6 @@ private class AssistantSessionViewOwner :
@Composable
private fun AssistantSessionSurface(
expanded: Boolean,
locked: Boolean,
screenContext: AssistantScreenContextUi,
onExpandedChange: (Boolean) -> Unit,
onCancel: () -> Unit,
@@ -527,8 +471,7 @@ private fun AssistantSessionSurface(
onOpenFullVoice: () -> Unit,
onSurfaceBoundsChanged: (android.graphics.Rect) -> Unit,
) {
val rawSnapshot by AssistantSessionState.snapshot.collectAsState()
val snapshot = assistantSnapshotForPresentation(rawSnapshot, locked)
val snapshot by AssistantSessionState.snapshot.collectAsState()
val status = assistantStatus(snapshot.phase)
val transmittedScreenContext = if (snapshot.screenContextSupported) {
screenContext
@@ -707,13 +650,6 @@ private fun ExpandedAssistantSurface(
color = MaterialTheme.colorScheme.onSurface,
)
}
snapshot.notice?.let { notice ->
Text(
text = assistantNoticeText(notice),
color = MaterialTheme.colorScheme.onSurfaceVariant,
style = MaterialTheme.typography.bodyMedium,
)
}
snapshot.error?.let { error ->
Text(
text = error,
@@ -960,11 +896,5 @@ private fun assistantStatus(phase: AssistantSessionPhase): String = when (phase)
private fun compactAssistantText(snapshot: AssistantSessionSnapshot): String =
snapshot.transcript?.takeIf { it.isNotBlank() }
?: snapshot.response.takeIf { it.isNotBlank() }
?: snapshot.notice?.let { assistantNoticeText(it) }
?: snapshot.error?.takeIf { it.isNotBlank() }
?: assistantStatus(snapshot.phase)
@Composable
private fun assistantNoticeText(notice: AssistantSessionNotice): String = when (notice) {
AssistantSessionNotice.NoSpeech -> stringResource(R.string.voice_no_speech_try_again)
}
@@ -240,25 +240,6 @@ class HermesProcessRuntime internal constructor(
}
}
fun republishAssistantSnapshot(activationId: String) {
val snapshot = synchronized(activationLock) {
if (currentActivationId != activationId ||
_initializationState.value != HermesRuntimeInitializationState.Ready
) {
null
} else {
binder.assistantSnapshot.value
}
} ?: return
if (snapshot.phase != com.hermesandroid.relay.assistant.AssistantSessionPhase.Closed) {
com.hermesandroid.relay.assistant.AssistantSessionProtocol.publish(
application,
activationId,
snapshot,
)
}
}
fun recordAssistantHeartbeat(
activationId: String,
nowElapsedMs: Long = SystemClock.elapsedRealtime(),
@@ -376,9 +376,7 @@ internal class HermesRuntimeBinder(
if (!AssistantAppSessionState.active.value) return@collect
if (state.voiceMode) AssistantAppSessionState.markVoiceStarted()
if (state.voiceMode || AssistantAppSessionState.hasVoiceStarted()) {
state.assistantActivationId?.let { activationId ->
AssistantSessionProtocol.publish(application, activationId, snapshot)
}
AssistantSessionProtocol.publish(application, snapshot)
}
}
}
@@ -52,7 +52,6 @@ import com.hermesandroid.relay.voice.VoiceCommandInterpreter
import com.hermesandroid.relay.voice.SpokenInterruptionLatch
import com.hermesandroid.relay.voice.voiceInterfaceContextPrompt
import com.hermesandroid.relay.assistant.assistantContextStore
import com.hermesandroid.relay.assistant.AssistantSessionNotice
import com.hermesandroid.relay.assistant.buildAssistantVoiceTurnPayload
// === PHASE3-voice-intents: voice→bridge intent routing ===
import com.hermesandroid.relay.voice.IntentResult
@@ -127,25 +126,6 @@ internal fun voiceSubmissionRetryState(state: VoiceUiState): VoiceUiState = stat
error = null,
)
internal fun voiceNoSpeechState(state: VoiceUiState): VoiceUiState = state.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
transcribedText = null,
error = null,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
internal fun voiceCaptureCancellationState(
state: VoiceUiState,
notice: AssistantSessionNotice? = null,
): VoiceUiState = state.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
assistantNotice = notice,
)
internal data class AssistantContextTurnDisposition(
val retireForLaterTurns: Boolean,
val consumeOnTransportAcceptance: Boolean,
@@ -348,10 +328,6 @@ data class VoiceUiState(
val responseText: String = "",
/** Human-readable error surfaced in the overlay. */
val error: String? = null,
/** Content-free retry status safe for the system Assistant surface. */
val assistantNotice: AssistantSessionNotice? = null,
/** Stable owner for cross-process Assistant status; null for ordinary voice. */
val assistantActivationId: String? = null,
/** Currently-selected interaction mode. */
val interactionMode: InteractionMode = InteractionMode.TapToTalk,
/**
@@ -464,7 +440,6 @@ internal fun voiceSessionExitState(state: VoiceUiState): VoiceUiState =
transcribedText = null,
responseText = "",
error = null,
assistantNotice = null,
destructiveCountdown = null,
hermesConfirmation = null,
handoffStatus = null,
@@ -1655,7 +1630,6 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
state = VoiceState.Idle,
outputAudioActive = false,
error = null,
assistantActivationId = activationId,
hermesConfirmation = null,
backgroundRun = if (orphanedRun != null) null else it.backgroundRun,
)
@@ -2093,7 +2067,6 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
state = VoiceState.Listening,
outputAudioActive = false,
error = null,
assistantNotice = null,
responseText = "",
// v0.4.1 — fresh turn, drop any stale JIT permission chip
// from the previous dispatch.
@@ -2203,11 +2176,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
private fun shouldDiscardVoiceCaptureBeforeStop(durationMs: Long): Boolean =
durationMs < MIN_VOICE_CAPTURE_DURATION_MS
private fun cancelListeningWithoutProcessing(
title: String,
detail: String? = null,
notice: AssistantSessionNotice? = null,
) {
private fun cancelListeningWithoutProcessing(title: String, detail: String? = null) {
responseInterruptedForVoiceCommand = false
silenceWatchdogJob?.cancel()
silenceWatchdogJob = null
@@ -2219,7 +2188,9 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
title = title,
detail = detail,
)
_uiState.update { voiceCaptureCancellationState(it, notice) }
_uiState.update {
it.copy(state = VoiceState.Idle, amplitude = 0f, outputAudioActive = false)
}
}
/** Reconcile microphone state after the Activity returns to foreground. */
@@ -2356,7 +2327,6 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
cancelListeningWithoutProcessing(
title = getApplication<Application>().getString(R.string.voice_status_no_speech),
detail = "No speech within ${IDLE_NO_SPEECH_MS / 1000}s",
notice = AssistantSessionNotice.NoSpeech,
)
return@launch
}
@@ -6527,7 +6497,13 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
detail = detail,
)
_uiState.update {
voiceNoSpeechState(it)
it.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
error = null,
transcribedText = null,
)
}
Toast.makeText(context, message, Toast.LENGTH_SHORT).show()
}
@@ -65,122 +65,6 @@ class AssistantSessionProtocolTest {
assertEquals("Microphone unavailable", error.error)
}
@Test
fun idleNoSpeech_isAVisibleRetryNotice() {
val snapshot = AssistantSessionProtocol.snapshotFromVoiceState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Idle,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
)
assertEquals(AssistantSessionPhase.Idle, snapshot.phase)
assertEquals(AssistantSessionNotice.NoSpeech, snapshot.notice)
assertNull(snapshot.error)
}
@Test
fun lockedPresentation_redactsConversationButKeepsGenericNotice() {
val presented = assistantSnapshotForPresentation(
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
transcript = "private request",
response = "private response",
notice = AssistantSessionNotice.NoSpeech,
error = "private route detail",
),
locked = true,
)
assertNull(presented.transcript)
assertEquals("", presented.response)
assertNull(presented.error)
assertEquals(AssistantSessionNotice.NoSpeech, presented.notice)
}
@Test
fun unlockedPresentation_restoresConversationContent() {
val snapshot = AssistantSessionSnapshot(
phase = AssistantSessionPhase.Speaking,
transcript = "request",
response = "response",
)
assertEquals(snapshot, assistantSnapshotForPresentation(snapshot, locked = false))
}
@Test
fun liveKeyguardState_overridesLaunchFallbackInBothDirections() {
assertTrue(
assistantPresentationLocked(
currentKeyguardLocked = true,
fallbackLocked = false,
)
)
assertFalse(
assistantPresentationLocked(
currentKeyguardLocked = false,
fallbackLocked = true,
)
)
}
@Test
fun statusSnapshots_areFencedToTheCurrentActivation() {
assertTrue(assistantSnapshotMatchesActivation("activation-b", "activation-b"))
assertFalse(assistantSnapshotMatchesActivation("activation-b", "activation-a"))
assertFalse(assistantSnapshotMatchesActivation("activation-b", null))
assertFalse(assistantSnapshotMatchesActivation(null, "activation-b"))
}
@Test
fun voiceStateRetainsItsOwningActivationAcrossLaterRuntimeChanges() {
val activationA = VoiceUiState(
voiceMode = true,
state = VoiceState.Listening,
assistantActivationId = "activation-a",
)
val currentRuntimeActivation = "activation-b"
assertEquals("activation-a", activationA.assistantActivationId)
assertFalse(
assistantSnapshotMatchesActivation(
expectedActivationId = currentRuntimeActivation,
receivedActivationId = activationA.assistantActivationId,
)
)
}
@Test
fun terminalVoiceStateRetainsActivationForClosedPublication() {
val exited = com.hermesandroid.relay.viewmodel.voiceSessionExitState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Speaking,
assistantActivationId = "activation-a",
)
)
assertFalse(exited.voiceMode)
assertEquals("activation-a", exited.assistantActivationId)
assertEquals(
AssistantSessionPhase.Closed,
AssistantSessionProtocol.snapshotFromVoiceState(exited).phase,
)
}
@Test
fun subsequentOrdinaryVoiceEntryClearsPreviousAssistantOwner() {
val ordinaryEntry = VoiceUiState(
voiceMode = true,
state = VoiceState.Idle,
assistantActivationId = null,
)
assertNull(ordinaryEntry.assistantActivationId)
}
@Test
fun persistedSessionMarker_expiresAfterBoundedRecoveryWindow() {
val now = 2_000_000L
@@ -1,9 +1,6 @@
package com.hermesandroid.relay.viewmodel
import com.hermesandroid.relay.assistant.AssistantSessionNotice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
@@ -23,35 +20,4 @@ class VoiceCaptureGuardTest {
fun acceptsSettledShortUtterance() {
assertFalse(shouldDiscardVoiceCapture(durationMs = 420L, pcmBytes = 12_000))
}
@Test
fun noSpeechReturnsToRetryableIdleWithDurableFeedback() {
val state = voiceNoSpeechState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Transcribing,
transcribedText = "stale",
)
)
assertEquals(VoiceState.Idle, state.state)
assertEquals(AssistantSessionNotice.NoSpeech, state.assistantNotice)
assertNull(state.error)
assertNull(state.transcribedText)
assertTrue(state.voiceMode)
}
@Test
fun unrelatedCaptureCancellationDoesNotClaimNoSpeech() {
val state = voiceCaptureCancellationState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Listening,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
)
assertEquals(VoiceState.Idle, state.state)
assertNull(state.assistantNotice)
}
}
-5
View File
@@ -2495,8 +2495,6 @@ boundary.
- Activation heartbeats let the main runtime clean up after assistant-process loss.
Finish and show-failure paths clear pending/watchdog state, while Full Voice
explicitly transfers ownership before the session overlay stops heartbeats.
A recreated session process requests the current activation-fenced voice
snapshot rather than treating its empty local state as authoritative.
- Connection, chat, and voice runtime ownership is application-lifetime in the
main process rather than Activity-owned. The assistant service may initialize
that graph and start a turn while no Activity exists; the app UI later binds
@@ -2510,9 +2508,6 @@ boundary.
- Background and locked-screen invocation is mediated by Android's selected
assistant UI/session rather than an ordinary background Activity launch.
- Locked assistant UI exposes only generic phase and retry status. Transcript,
response, route-specific errors, diagnostics, and screen context remain hidden
until the device is unlocked; no-speech retry copy is deliberately content-free.
- Users can leave Hermes selected for gesture/power-button invocation while
turning continuous KWS off, or remove Hermes through Android's Assistant
settings.
+1 -6
View File
@@ -1015,10 +1015,7 @@ utilities.
application-lifetime owner, allowing assistant activation to start cold
without constructing or foregrounding `MainActivity`; full Voice later binds
that same runtime. Cancel, error, app/process recreation, and session finish
use the same scoped protocol. Recreated session UI requests an
activation-fenced snapshot from the app runtime, and capture/no-speech exits
retain a generic retry notice instead of collapsing to an unexplained Ready
state. The
use the same scoped protocol. The
wake recorder is released before the established voice recorder opens, and
assistant listening resumes only after the session exits. This mode is
mutually exclusive with the experimental notification-based foreground
@@ -1049,8 +1046,6 @@ utilities.
Realtime Agent sessions do not claim inclusion. The mic control follows the
active voice state, close remains separate, and **Open full voice** explicitly
transfers ownership so assistant-process cleanup cannot cancel the main-app flow.
While keyguard is active, the surface keeps only generic phase and retry copy;
transcript, response, route-specific errors, and screen context remain hidden.
- Stable voice integrates with `ChatViewModel` by **observing** `messages: StateFlow`; transcribed text goes through normal `chatVm.sendMessage(text)` so voice utterances appear as regular user messages in chat history. Experimental Realtime Agent creates a mirrored chat turn and applies broker events directly so tool state, transcript text, assistant deltas, and final responses appear without leaving voice mode.
- `VoiceModeOverlay` — full-screen UI with the MorphingSphere at 60% height in `voiceMode=true`, transcribed + response text, mic button supporting Tap / Hold / Continuous interaction modes.
- The optional `SYSTEM_ALERT_WINDOW` Voice control is user-invoked from an
+1 -1
View File
@@ -31,7 +31,7 @@ play-publisher = "4.1.1"
media3 = "1.11.0"
androidVad = "2.0.10"
sherpaOnnx = "v1.13.4"
onnxRuntime = "1.29.0"
onnxRuntime = "1.27.0"
spatialsdk = "0.13.2"
play-app-update = "2.1.0"
+306
View File
@@ -0,0 +1,306 @@
#!/usr/bin/env python3
"""Verify that packaged Android JNI consumers match their shared ONNX Runtime.
The APK is the authority for this check. Gradle can resolve multiple AARs that
contain the same native filename, and ``pickFirst`` alone does not prove that
the selected runtime exports the symbol version required by sherpa's JNI.
"""
from __future__ import annotations
import argparse
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
import re
import struct
import sys
import zipfile
ORT_ENTRY_POINT = "OrtGetApiBase"
RUNTIME_LIBRARY = "libonnxruntime.so"
REQUIRED_CONSUMER = "libsherpa-onnx-jni.so"
ORT_JAVA_CONSUMER = "libonnxruntime4j_jni.so"
SUPPORTED_ABIS = {"arm64-v8a", "armeabi-v7a", "x86", "x86_64"}
LIBRARY_PATH = re.compile(
r"^(?:base/)?lib/(?P<abi>[^/]+)/(?P<library>[^/]+\.so)$"
)
@dataclass(frozen=True)
class Section:
section_type: int
offset: int
size: int
link: int
entry_size: int
@dataclass(frozen=True)
class VersionedSymbol:
defined: bool
version: str | None
def _unpack(fmt: str, data: bytes, offset: int) -> tuple[int, ...]:
size = struct.calcsize(fmt)
if offset < 0 or offset + size > len(data):
raise ValueError("ELF structure extends beyond the file")
return struct.unpack_from(fmt, data, offset)
def _cstring(data: bytes, offset: int) -> str:
if offset < 0 or offset >= len(data):
raise ValueError("ELF string offset is outside its string table")
end = data.find(b"\0", offset)
if end < 0:
raise ValueError("ELF string is not NUL-terminated")
return data[offset:end].decode("utf-8", errors="replace")
def _elf_layout(data: bytes) -> tuple[str, bool, list[Section]]:
if len(data) < 16 or data[:4] != b"\x7fELF":
raise ValueError("not an ELF file")
elf_class = data[4]
byte_order = data[5]
if elf_class not in (1, 2) or byte_order not in (1, 2):
raise ValueError("unsupported ELF class or byte order")
endian = "<" if byte_order == 1 else ">"
is_64_bit = elf_class == 2
if is_64_bit:
header = _unpack(endian + "HHIQQQIHHHHHH", data, 16)
section_offset, section_entry_size, section_count = header[5], header[10], header[11]
section_format = endian + "IIQQQQIIQQ"
else:
header = _unpack(endian + "HHIIIIIHHHHHH", data, 16)
section_offset, section_entry_size, section_count = header[5], header[10], header[11]
section_format = endian + "IIIIIIIIII"
minimum_entry_size = struct.calcsize(section_format)
if section_entry_size < minimum_entry_size:
raise ValueError("ELF section-header entry is too small")
sections: list[Section] = []
for index in range(section_count):
fields = _unpack(section_format, data, section_offset + index * section_entry_size)
sections.append(
Section(
section_type=fields[1],
offset=fields[4],
size=fields[5],
link=fields[6],
entry_size=fields[9],
)
)
return endian, is_64_bit, sections
def _section_data(data: bytes, section: Section) -> bytes:
end = section.offset + section.size
if section.offset < 0 or end > len(data):
raise ValueError("ELF section extends beyond the file")
return data[section.offset:end]
def _version_names(
data: bytes,
endian: str,
sections: list[Section],
) -> dict[int, str]:
names: dict[int, str] = {}
for section in sections:
if section.section_type not in (0x6FFFFFFD, 0x6FFFFFFE):
continue
if section.link >= len(sections):
raise ValueError("ELF version section has an invalid string-table link")
strings = _section_data(data, sections[section.link])
cursor = 0
while cursor < section.size:
base = section.offset + cursor
if section.section_type == 0x6FFFFFFD: # SHT_GNU_verdef
fields = _unpack(endian + "HHHHIII", data, base)
version_index, aux_offset, next_offset = fields[2], fields[5], fields[6]
name_offset, _ = _unpack(endian + "II", data, base + aux_offset)
names[version_index] = _cstring(strings, name_offset)
else: # SHT_GNU_verneed
fields = _unpack(endian + "HHIII", data, base)
count, aux_offset, next_offset = fields[1], fields[3], fields[4]
aux_cursor = base + aux_offset
for _ in range(count):
aux = _unpack(endian + "IHHII", data, aux_cursor)
names[aux[2] & 0x7FFF] = _cstring(strings, aux[3])
if aux[4] == 0:
break
aux_cursor += aux[4]
if next_offset == 0:
break
cursor += next_offset
return names
def read_versioned_symbols(data: bytes) -> dict[str, list[VersionedSymbol]]:
endian, is_64_bit, sections = _elf_layout(data)
version_names = _version_names(data, endian, sections)
symbols: dict[str, list[VersionedSymbol]] = defaultdict(list)
for dynsym_index, dynsym in enumerate(sections):
if dynsym.section_type != 11: # SHT_DYNSYM
continue
if dynsym.link >= len(sections):
raise ValueError("ELF dynamic-symbol table has an invalid string-table link")
strings = _section_data(data, sections[dynsym.link])
symbol_format = endian + ("IBBHQQ" if is_64_bit else "IIIBBH")
symbol_size = dynsym.entry_size or struct.calcsize(symbol_format)
symbol_count = dynsym.size // symbol_size
versions: tuple[int, ...] = ()
for section in sections:
if section.section_type == 0x6FFFFFFF and section.link == dynsym_index:
raw_versions = _section_data(data, section)
versions = struct.unpack(endian + f"{len(raw_versions) // 2}H", raw_versions)
break
for index in range(symbol_count):
fields = _unpack(symbol_format, data, dynsym.offset + index * symbol_size)
name_offset = fields[0]
section_index = fields[3] if is_64_bit else fields[5]
name = _cstring(strings, name_offset)
if not name:
continue
version_index = (versions[index] & 0x7FFF) if index < len(versions) else 0
symbols[name].append(
VersionedSymbol(
defined=section_index != 0,
version=version_names.get(version_index),
)
)
return symbols
def _single_symbol_version(
blob: bytes,
*,
defined: bool,
context: str,
) -> str:
matches = [
symbol.version
for symbol in read_versioned_symbols(blob).get(ORT_ENTRY_POINT, [])
if symbol.defined == defined
]
if not matches:
role = "export" if defined else "requirement"
raise ValueError(f"{context} has no {ORT_ENTRY_POINT} {role}")
versions = set(matches)
if None in versions:
raise ValueError(f"{context} uses an unversioned {ORT_ENTRY_POINT} symbol")
if len(versions) != 1:
raise ValueError(f"{context} has ambiguous {ORT_ENTRY_POINT} versions: {sorted(versions)}")
return next(iter(versions)) # type: ignore[return-value]
def check_artifact(path: Path, expected_abis: set[str] | None = None) -> list[str]:
failures: list[str] = []
with zipfile.ZipFile(path) as archive:
libraries: dict[str, dict[str, list[zipfile.ZipInfo]]] = defaultdict(
lambda: defaultdict(list)
)
for info in archive.infolist():
match = LIBRARY_PATH.fullmatch(info.filename)
if match:
libraries[match.group("abi")][match.group("library")].append(info)
if not libraries:
return [f"{path}: no packaged native libraries found"]
expected = SUPPORTED_ABIS if expected_abis is None else expected_abis
actual = set(libraries)
if actual != expected:
failures.append(
f"{path.name}: packaged ABI set is {sorted(actual)}, expected {sorted(expected)}"
)
for abi, by_name in sorted(libraries.items()):
for required in (RUNTIME_LIBRARY, REQUIRED_CONSUMER, ORT_JAVA_CONSUMER):
count = len(by_name.get(required, []))
if count != 1:
failures.append(
f"{path.name} [{abi}]: expected exactly one {required}, found {count}"
)
if failures and (
len(by_name.get(RUNTIME_LIBRARY, [])) != 1
or len(by_name.get(REQUIRED_CONSUMER, [])) != 1
or len(by_name.get(ORT_JAVA_CONSUMER, [])) != 1
):
continue
try:
runtime_version = _single_symbol_version(
archive.read(by_name[RUNTIME_LIBRARY][0]),
defined=True,
context=f"{path.name} [{abi}] {RUNTIME_LIBRARY}",
)
for consumer in (REQUIRED_CONSUMER, ORT_JAVA_CONSUMER):
consumer_version = _single_symbol_version(
archive.read(by_name[consumer][0]),
defined=False,
context=f"{path.name} [{abi}] {consumer}",
)
if runtime_version != consumer_version:
failures.append(
f"{path.name} [{abi}]: {consumer} requires "
f"{ORT_ENTRY_POINT}@{consumer_version}, but {RUNTIME_LIBRARY} exports "
f"{ORT_ENTRY_POINT}@{runtime_version}"
)
except ValueError as error:
failures.append(str(error))
return failures
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"artifact",
nargs="+",
type=Path,
help="APK or AAB artifact to inspect.",
)
parser.add_argument(
"--expected-abi",
action="append",
default=[],
help=(
"Expected packaged ABI; repeat to override the standard four-ABI set "
"for a deliberate -Phermes.devAbi build."
),
)
args = parser.parse_args()
failures: list[str] = []
for artifact in args.artifact:
if not artifact.is_file():
failures.append(f"artifact does not exist: {artifact}")
continue
try:
expected_abis = set(args.expected_abi) or SUPPORTED_ABIS
failures.extend(check_artifact(artifact, expected_abis))
except (OSError, ValueError, zipfile.BadZipFile) as error:
failures.append(f"{artifact}: {error}")
if failures:
print("Android native compatibility check failed:", file=sys.stderr)
for failure in failures:
print(f" {failure}", file=sys.stderr)
return 1
print(
"Android native compatibility check passed "
f"({len(args.artifact)} artifact(s), {ORT_ENTRY_POINT} symbol versions aligned)"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,164 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import struct
import sys
import tempfile
import unittest
import zipfile
SCRIPT = Path(__file__).resolve().parents[1] / "check-android-native-compat.py"
SPEC = importlib.util.spec_from_file_location("check_android_native_compat", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
native_compat = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = native_compat
SPEC.loader.exec_module(native_compat)
def elf_with_ort_symbol(*, defined: bool, version: str) -> bytes:
"""Build a minimal ELF64 containing a versioned OrtGetApiBase dynsym."""
symbol_name = b"OrtGetApiBase"
version_name = version.encode("ascii")
dependency_name = b"libonnxruntime.so"
strings = b"\0" + symbol_name + b"\0" + version_name + b"\0" + dependency_name + b"\0"
symbol_offset = 1
version_offset = symbol_offset + len(symbol_name) + 1
dependency_offset = version_offset + len(version_name) + 1
dynsym = b"\0" * 24 + struct.pack(
"<IBBHQQ",
symbol_offset,
0x12,
0,
1 if defined else 0,
0,
0,
)
versym = struct.pack("<HH", 0, 2)
if defined:
version_section_type = 0x6FFFFFFD
version_data = struct.pack("<HHHHIII", 1, 0, 2, 1, 0, 20, 0)
version_data += struct.pack("<II", version_offset, 0)
else:
version_section_type = 0x6FFFFFFE
version_data = struct.pack("<HHIII", 1, 1, dependency_offset, 16, 0)
version_data += struct.pack("<IHHII", 0, 0, 2, version_offset, 0)
section_blobs = [b"", strings, dynsym, versym, version_data]
offsets: list[int] = []
image = bytearray(b"\0" * 64)
for blob in section_blobs:
while len(image) % 8:
image.append(0)
offsets.append(len(image))
image.extend(blob)
while len(image) % 8:
image.append(0)
section_header_offset = len(image)
section_headers = [
(0, 0, 0, 0, offsets[0], 0, 0, 0, 0, 0),
(0, 3, 0, 0, offsets[1], len(strings), 0, 0, 1, 0),
(0, 11, 0, 0, offsets[2], len(dynsym), 1, 0, 8, 24),
(0, 0x6FFFFFFF, 0, 0, offsets[3], len(versym), 2, 0, 2, 2),
(0, version_section_type, 0, 0, offsets[4], len(version_data), 1, 0, 4, 0),
]
for header in section_headers:
image.extend(struct.pack("<IIQQQQIIQQ", *header))
ident = b"\x7fELF" + bytes((2, 1, 1, 0)) + b"\0" * 8
header = struct.pack(
"<16sHHIQQQIHHHHHH",
ident,
3,
183,
1,
0,
0,
section_header_offset,
0,
64,
0,
0,
64,
len(section_headers),
0,
)
image[:64] = header
return bytes(image)
def write_artifact(path: Path, *, runtime_version: str, abis: set[str]) -> None:
with zipfile.ZipFile(path, "w") as archive:
for abi in abis:
prefix = f"lib/{abi}/"
archive.writestr(
prefix + native_compat.RUNTIME_LIBRARY,
elf_with_ort_symbol(defined=True, version=runtime_version),
)
for consumer in (
native_compat.REQUIRED_CONSUMER,
native_compat.ORT_JAVA_CONSUMER,
):
archive.writestr(
prefix + consumer,
elf_with_ort_symbol(defined=False, version="VERS_1.27.0"),
)
class AndroidNativeCompatTest(unittest.TestCase):
def test_parses_gnu_definition_and_requirement_versions(self) -> None:
provider = native_compat.read_versioned_symbols(
elf_with_ort_symbol(defined=True, version="VERS_1.27.0")
)
consumer = native_compat.read_versioned_symbols(
elf_with_ort_symbol(defined=False, version="VERS_1.27.0")
)
self.assertEqual(
[native_compat.VersionedSymbol(defined=True, version="VERS_1.27.0")],
provider[native_compat.ORT_ENTRY_POINT],
)
self.assertEqual(
[native_compat.VersionedSymbol(defined=False, version="VERS_1.27.0")],
consumer[native_compat.ORT_ENTRY_POINT],
)
def test_accepts_aligned_runtime_and_both_consumers(self) -> None:
with tempfile.TemporaryDirectory() as directory:
artifact = Path(directory) / "aligned.apk"
write_artifact(
artifact,
runtime_version="VERS_1.27.0",
abis=native_compat.SUPPORTED_ABIS,
)
self.assertEqual([], native_compat.check_artifact(artifact))
def test_rejects_runtime_symbol_version_mismatch(self) -> None:
with tempfile.TemporaryDirectory() as directory:
artifact = Path(directory) / "mismatch.apk"
write_artifact(
artifact,
runtime_version="VERS_1.29.0",
abis=native_compat.SUPPORTED_ABIS,
)
failures = native_compat.check_artifact(artifact)
self.assertTrue(any("requires OrtGetApiBase@VERS_1.27.0" in item for item in failures))
self.assertTrue(any(native_compat.ORT_JAVA_CONSUMER in item for item in failures))
def test_rejects_missing_supported_abi(self) -> None:
with tempfile.TemporaryDirectory() as directory:
artifact = Path(directory) / "missing-abi.apk"
write_artifact(
artifact,
runtime_version="VERS_1.27.0",
abis={"arm64-v8a"},
)
failures = native_compat.check_artifact(artifact)
self.assertTrue(any("packaged ABI set" in item for item in failures))
if __name__ == "__main__":
unittest.main()