Compare commits

...
Author SHA1 Message Date
Bailey Dixon 578c074797 fix(android): surface assistant capture recovery 2026-08-29 13:37:35 -04:00
11 changed files with 351 additions and 26 deletions
+1
View File
@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
### Fixed
- **The visible Android Sphere keeps its smooth procedural motion across startup and chat.** Backgrounded and motion-disabled surfaces remain still without reducing foreground animation to a stepped ambient pulse.
- **Android Assistant sessions explain when no speech was captured instead of appearing stuck at Ready.** Retry feedback survives the separate system overlay process, recreated session UI requests the current turn state, and locked sessions keep transcript, response, and technical error text private.
### Removed
+6 -1
View File
@@ -1300,7 +1300,12 @@ and whether the agent is waiting on the user.
permissions; exercise compact, expanded, collapsed, and full-Voice handoff
states, background tap-through, rotation and insets, cancel/back, microphone
denial, network failure, process kill/recreation, and wake→voice→wake
resumption. Measure idle battery drain because third-party assistants do not
resumption. For background and keyguard capture, record `AudioRecord`, AppOps,
and foreground-service state: the user-installed app owns capture outside the
separate session process, so confirm whether the selected Assistant role is
sufficient on each target OS or whether activation needs an explicit,
activation-scoped microphone foreground-service lease. Measure idle battery
drain because third-party assistants do not
receive Google's dedicated low-power hotword hardware.
- **Audio quality guardrails** — normalize output volume across realtime and
@@ -39,14 +39,38 @@ enum class AssistantSessionPhase {
Closed,
}
enum class AssistantSessionNotice {
NoSpeech,
}
data class AssistantSessionSnapshot(
val phase: AssistantSessionPhase = AssistantSessionPhase.Launching,
val transcript: String? = null,
val response: String = "",
val notice: AssistantSessionNotice? = null,
val error: String? = null,
val screenContextSupported: Boolean = false,
)
internal fun assistantSnapshotForPresentation(
snapshot: AssistantSessionSnapshot,
locked: Boolean,
): AssistantSessionSnapshot = if (locked) {
snapshot.copy(
transcript = null,
response = "",
error = null,
screenContextSupported = false,
)
} else {
snapshot
}
internal fun assistantSnapshotMatchesActivation(
expectedActivationId: String?,
receivedActivationId: String?,
): Boolean = expectedActivationId != null && expectedActivationId == receivedActivationId
object AssistantRole {
fun status(context: Context): AssistantRoleStatus {
val component = ComponentName(context, HermesVoiceInteractionService::class.java)
@@ -209,6 +233,7 @@ object AssistantSessionProtocol {
onFailure = { failure ->
publish(
application,
activation.id,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
@@ -219,13 +244,19 @@ object AssistantSessionProtocol {
return true
}
fun publish(context: Context, snapshot: AssistantSessionSnapshot) {
fun publish(
context: Context,
activationId: String,
snapshot: AssistantSessionSnapshot,
) {
context.sendBroadcast(
Intent(context, AssistantSessionStateReceiver::class.java).apply {
action = ACTION_STATUS
putExtra(EXTRA_ACTIVATION_ID, activationId)
putExtra(EXTRA_PHASE, snapshot.phase.name)
putExtra(EXTRA_TRANSCRIPT, snapshot.transcript)
putExtra(EXTRA_RESPONSE, snapshot.response)
putExtra(EXTRA_NOTICE, snapshot.notice?.name)
putExtra(EXTRA_ERROR, snapshot.error)
putExtra(EXTRA_SCREEN_CONTEXT_SUPPORTED, snapshot.screenContextSupported)
}
@@ -238,10 +269,6 @@ object AssistantSessionProtocol {
}
}
fun publish(context: Context, state: VoiceUiState) {
publish(context, snapshotFromVoiceState(state))
}
internal fun snapshotFromVoiceState(state: VoiceUiState): AssistantSessionSnapshot {
val phase = when {
!state.voiceMode -> AssistantSessionPhase.Closed
@@ -256,7 +283,10 @@ object AssistantSessionProtocol {
phase = phase,
transcript = state.transcribedText?.take(MAX_SESSION_TEXT_CHARS),
response = state.responseText.take(MAX_SESSION_TEXT_CHARS),
error = state.error?.take(MAX_SESSION_ERROR_CHARS),
notice = state.assistantNotice,
error = state.error
?.takeIf { phase == AssistantSessionPhase.Error }
?.take(MAX_SESSION_ERROR_CHARS),
)
}
@@ -350,6 +380,9 @@ object AssistantSessionProtocol {
phase = phase,
transcript = intent.getStringExtra(EXTRA_TRANSCRIPT),
response = intent.getStringExtra(EXTRA_RESPONSE).orEmpty(),
notice = intent.getStringExtra(EXTRA_NOTICE)?.let { raw ->
runCatching { AssistantSessionNotice.valueOf(raw) }.getOrNull()
},
error = intent.getStringExtra(EXTRA_ERROR),
screenContextSupported = intent.getBooleanExtra(
EXTRA_SCREEN_CONTEXT_SUPPORTED,
@@ -360,24 +393,33 @@ object AssistantSessionProtocol {
private const val MAX_SESSION_TEXT_CHARS = 4_000
private const val MAX_SESSION_ERROR_CHARS = 1_000
private const val EXTRA_NOTICE = "notice"
}
object AssistantSessionState {
private val _snapshot = MutableStateFlow(AssistantSessionSnapshot())
val snapshot: StateFlow<AssistantSessionSnapshot> = _snapshot.asStateFlow()
@Volatile private var activationId: String? = null
internal fun update(snapshot: AssistantSessionSnapshot) {
internal fun update(receivedActivationId: String?, snapshot: AssistantSessionSnapshot) {
if (!assistantSnapshotMatchesActivation(activationId, receivedActivationId)) return
_snapshot.value = snapshot
}
internal fun reset() {
internal fun reset(activationId: String) {
this.activationId = activationId
_snapshot.value = AssistantSessionSnapshot()
}
}
class AssistantSessionStateReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
AssistantSessionState.update(AssistantSessionProtocol.readSnapshot(intent))
AssistantSessionState.update(
receivedActivationId = intent.getStringExtra(
AssistantSessionProtocol.EXTRA_ACTIVATION_ID
),
snapshot = AssistantSessionProtocol.readSnapshot(intent),
)
}
}
@@ -423,6 +465,7 @@ class AssistantSessionLifecycleReceiver : BroadcastReceiver() {
onFailure = { failure ->
AssistantSessionProtocol.publish(
application,
id,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
@@ -430,6 +473,7 @@ class AssistantSessionLifecycleReceiver : BroadcastReceiver() {
)
},
)
application.runtime.republishAssistantSnapshot(id)
return
}
if (AssistantSessionProtocol.isStartAction(intent.action)) {
@@ -3,6 +3,11 @@ package com.hermesandroid.relay.assistant
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.drawable.ColorDrawable
import android.app.KeyguardManager
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.os.Bundle
import android.service.voice.VoiceInteractionSession
import android.service.voice.VoiceInteractionSessionService
@@ -67,6 +72,7 @@ import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.LifecycleRegistry
@@ -108,6 +114,11 @@ internal fun shouldCancelVoiceWhenSessionUiEnds(
presentation: AssistantSessionPresentation,
): Boolean = presentation == AssistantSessionPresentation.Overlay
internal fun assistantPresentationLocked(
currentKeyguardLocked: Boolean?,
fallbackLocked: Boolean,
): Boolean = currentKeyguardLocked ?: fallbackLocked
private class HermesVoiceInteractionSession(
private val service: HermesVoiceInteractionSessionService,
) : VoiceInteractionSession(service) {
@@ -118,12 +129,19 @@ private class HermesVoiceInteractionSession(
private var surfaceExpanded by mutableStateOf(false)
private var activationId: String? = null
private var manualMic = false
private var keyguardLocked by mutableStateOf(false)
private var expectScreenContext: Boolean? = null
private var pendingSemantic = AssistantSemanticContext()
private var pendingScreenshot: ByteArray? = null
private var screenContextUi by mutableStateOf(AssistantScreenContextUi())
private val contextStore = assistantContextStore(service)
private var heartbeatJob: Job? = null
private var keyguardReceiverRegistered = false
private val keyguardReceiver = object : BroadcastReceiver() {
override fun onReceive(context: Context?, intent: Intent?) {
refreshKeyguardState()
}
}
init {
scope.launch {
@@ -139,6 +157,17 @@ private class HermesVoiceInteractionSession(
override fun onCreate() {
super.onCreate()
ContextCompat.registerReceiver(
service,
keyguardReceiver,
IntentFilter().apply {
addAction(Intent.ACTION_SCREEN_OFF)
addAction(Intent.ACTION_SCREEN_ON)
addAction(Intent.ACTION_USER_PRESENT)
},
ContextCompat.RECEIVER_NOT_EXPORTED,
)
keyguardReceiverRegistered = true
window.window?.apply {
setBackgroundDrawable(ColorDrawable(android.graphics.Color.TRANSPARENT))
clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND)
@@ -155,6 +184,7 @@ private class HermesVoiceInteractionSession(
PersistedHermesRelayTheme {
AssistantSessionSurface(
expanded = surfaceExpanded,
locked = keyguardLocked,
screenContext = screenContextUi,
onExpandedChange = { surfaceExpanded = it },
onCancel = { finishSession(cancelVoice = true) },
@@ -183,11 +213,22 @@ private class HermesVoiceInteractionSession(
override fun onShow(args: Bundle?, showFlags: Int) {
super.onShow(args, showFlags)
if (args?.getBoolean(HermesVoiceInteractionService.EXTRA_FROM_KEYGUARD, false) == true) {
refreshKeyguardState(
fallbackLocked = args?.getBoolean(
HermesVoiceInteractionService.EXTRA_FROM_KEYGUARD,
false,
) == true,
)
if (keyguardLocked) {
window.window?.addFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
} else {
window.window?.clearFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
}
setUiEnabled(true)
val startsNewLifecycle = presentation == AssistantSessionPresentation.Inactive
@@ -195,10 +236,10 @@ private class HermesVoiceInteractionSession(
if (!startsNewLifecycle) return
surfaceExpanded = false
AssistantSessionState.reset()
screenContextUi = AssistantScreenContextUi()
activationId = args?.getString(AssistantSessionProtocol.EXTRA_ACTIVATION_ID)
?: UUID.randomUUID().toString()
AssistantSessionState.reset(activationId!!)
manualMic = args?.getBoolean(AssistantSessionProtocol.EXTRA_MANUAL_MIC, false) ?: false
expectScreenContext = args?.getBoolean(
AssistantSessionProtocol.EXTRA_EXPECT_SCREEN_CONTEXT,
@@ -300,6 +341,10 @@ private class HermesVoiceInteractionSession(
pendingSemantic = AssistantSemanticContext()
pendingScreenshot = null
screenContextUi = AssistantScreenContextUi()
if (keyguardReceiverRegistered) {
runCatching { service.unregisterReceiver(keyguardReceiver) }
keyguardReceiverRegistered = false
}
viewOwner.stop()
scope.cancel()
super.onDestroy()
@@ -319,6 +364,7 @@ private class HermesVoiceInteractionSession(
)
}.onFailure {
AssistantSessionState.update(
activationId,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open the voice session.",
@@ -337,6 +383,7 @@ private class HermesVoiceInteractionSession(
setUiEnabled(false)
}.onFailure {
AssistantSessionState.update(
activationId,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open full voice.",
@@ -377,6 +424,14 @@ private class HermesVoiceInteractionSession(
}
}
private fun refreshKeyguardState(fallbackLocked: Boolean = keyguardLocked) {
keyguardLocked = assistantPresentationLocked(
currentKeyguardLocked = service.getSystemService(KeyguardManager::class.java)
?.isKeyguardLocked,
fallbackLocked = fallbackLocked,
)
}
@RequiresApi(android.os.Build.VERSION_CODES.Q)
private fun stageAssistState(state: AssistState) {
stageAssistData(state.assistStructure, state.assistContent)
@@ -463,6 +518,7 @@ private class AssistantSessionViewOwner :
@Composable
private fun AssistantSessionSurface(
expanded: Boolean,
locked: Boolean,
screenContext: AssistantScreenContextUi,
onExpandedChange: (Boolean) -> Unit,
onCancel: () -> Unit,
@@ -471,7 +527,8 @@ private fun AssistantSessionSurface(
onOpenFullVoice: () -> Unit,
onSurfaceBoundsChanged: (android.graphics.Rect) -> Unit,
) {
val snapshot by AssistantSessionState.snapshot.collectAsState()
val rawSnapshot by AssistantSessionState.snapshot.collectAsState()
val snapshot = assistantSnapshotForPresentation(rawSnapshot, locked)
val status = assistantStatus(snapshot.phase)
val transmittedScreenContext = if (snapshot.screenContextSupported) {
screenContext
@@ -650,6 +707,13 @@ private fun ExpandedAssistantSurface(
color = MaterialTheme.colorScheme.onSurface,
)
}
snapshot.notice?.let { notice ->
Text(
text = assistantNoticeText(notice),
color = MaterialTheme.colorScheme.onSurfaceVariant,
style = MaterialTheme.typography.bodyMedium,
)
}
snapshot.error?.let { error ->
Text(
text = error,
@@ -896,5 +960,11 @@ private fun assistantStatus(phase: AssistantSessionPhase): String = when (phase)
private fun compactAssistantText(snapshot: AssistantSessionSnapshot): String =
snapshot.transcript?.takeIf { it.isNotBlank() }
?: snapshot.response.takeIf { it.isNotBlank() }
?: snapshot.notice?.let { assistantNoticeText(it) }
?: snapshot.error?.takeIf { it.isNotBlank() }
?: assistantStatus(snapshot.phase)
@Composable
private fun assistantNoticeText(notice: AssistantSessionNotice): String = when (notice) {
AssistantSessionNotice.NoSpeech -> stringResource(R.string.voice_no_speech_try_again)
}
@@ -240,6 +240,25 @@ class HermesProcessRuntime internal constructor(
}
}
fun republishAssistantSnapshot(activationId: String) {
val snapshot = synchronized(activationLock) {
if (currentActivationId != activationId ||
_initializationState.value != HermesRuntimeInitializationState.Ready
) {
null
} else {
binder.assistantSnapshot.value
}
} ?: return
if (snapshot.phase != com.hermesandroid.relay.assistant.AssistantSessionPhase.Closed) {
com.hermesandroid.relay.assistant.AssistantSessionProtocol.publish(
application,
activationId,
snapshot,
)
}
}
fun recordAssistantHeartbeat(
activationId: String,
nowElapsedMs: Long = SystemClock.elapsedRealtime(),
@@ -376,7 +376,9 @@ internal class HermesRuntimeBinder(
if (!AssistantAppSessionState.active.value) return@collect
if (state.voiceMode) AssistantAppSessionState.markVoiceStarted()
if (state.voiceMode || AssistantAppSessionState.hasVoiceStarted()) {
AssistantSessionProtocol.publish(application, snapshot)
state.assistantActivationId?.let { activationId ->
AssistantSessionProtocol.publish(application, activationId, snapshot)
}
}
}
}
@@ -52,6 +52,7 @@ import com.hermesandroid.relay.voice.VoiceCommandInterpreter
import com.hermesandroid.relay.voice.SpokenInterruptionLatch
import com.hermesandroid.relay.voice.voiceInterfaceContextPrompt
import com.hermesandroid.relay.assistant.assistantContextStore
import com.hermesandroid.relay.assistant.AssistantSessionNotice
import com.hermesandroid.relay.assistant.buildAssistantVoiceTurnPayload
// === PHASE3-voice-intents: voice→bridge intent routing ===
import com.hermesandroid.relay.voice.IntentResult
@@ -126,6 +127,25 @@ internal fun voiceSubmissionRetryState(state: VoiceUiState): VoiceUiState = stat
error = null,
)
internal fun voiceNoSpeechState(state: VoiceUiState): VoiceUiState = state.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
transcribedText = null,
error = null,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
internal fun voiceCaptureCancellationState(
state: VoiceUiState,
notice: AssistantSessionNotice? = null,
): VoiceUiState = state.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
assistantNotice = notice,
)
internal data class AssistantContextTurnDisposition(
val retireForLaterTurns: Boolean,
val consumeOnTransportAcceptance: Boolean,
@@ -328,6 +348,10 @@ data class VoiceUiState(
val responseText: String = "",
/** Human-readable error surfaced in the overlay. */
val error: String? = null,
/** Content-free retry status safe for the system Assistant surface. */
val assistantNotice: AssistantSessionNotice? = null,
/** Stable owner for cross-process Assistant status; null for ordinary voice. */
val assistantActivationId: String? = null,
/** Currently-selected interaction mode. */
val interactionMode: InteractionMode = InteractionMode.TapToTalk,
/**
@@ -440,6 +464,7 @@ internal fun voiceSessionExitState(state: VoiceUiState): VoiceUiState =
transcribedText = null,
responseText = "",
error = null,
assistantNotice = null,
destructiveCountdown = null,
hermesConfirmation = null,
handoffStatus = null,
@@ -1630,6 +1655,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
state = VoiceState.Idle,
outputAudioActive = false,
error = null,
assistantActivationId = activationId,
hermesConfirmation = null,
backgroundRun = if (orphanedRun != null) null else it.backgroundRun,
)
@@ -2067,6 +2093,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
state = VoiceState.Listening,
outputAudioActive = false,
error = null,
assistantNotice = null,
responseText = "",
// v0.4.1 — fresh turn, drop any stale JIT permission chip
// from the previous dispatch.
@@ -2176,7 +2203,11 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
private fun shouldDiscardVoiceCaptureBeforeStop(durationMs: Long): Boolean =
durationMs < MIN_VOICE_CAPTURE_DURATION_MS
private fun cancelListeningWithoutProcessing(title: String, detail: String? = null) {
private fun cancelListeningWithoutProcessing(
title: String,
detail: String? = null,
notice: AssistantSessionNotice? = null,
) {
responseInterruptedForVoiceCommand = false
silenceWatchdogJob?.cancel()
silenceWatchdogJob = null
@@ -2188,9 +2219,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
title = title,
detail = detail,
)
_uiState.update {
it.copy(state = VoiceState.Idle, amplitude = 0f, outputAudioActive = false)
}
_uiState.update { voiceCaptureCancellationState(it, notice) }
}
/** Reconcile microphone state after the Activity returns to foreground. */
@@ -2327,6 +2356,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
cancelListeningWithoutProcessing(
title = getApplication<Application>().getString(R.string.voice_status_no_speech),
detail = "No speech within ${IDLE_NO_SPEECH_MS / 1000}s",
notice = AssistantSessionNotice.NoSpeech,
)
return@launch
}
@@ -6497,13 +6527,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
detail = detail,
)
_uiState.update {
it.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
error = null,
transcribedText = null,
)
voiceNoSpeechState(it)
}
Toast.makeText(context, message, Toast.LENGTH_SHORT).show()
}
@@ -65,6 +65,122 @@ class AssistantSessionProtocolTest {
assertEquals("Microphone unavailable", error.error)
}
@Test
fun idleNoSpeech_isAVisibleRetryNotice() {
val snapshot = AssistantSessionProtocol.snapshotFromVoiceState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Idle,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
)
assertEquals(AssistantSessionPhase.Idle, snapshot.phase)
assertEquals(AssistantSessionNotice.NoSpeech, snapshot.notice)
assertNull(snapshot.error)
}
@Test
fun lockedPresentation_redactsConversationButKeepsGenericNotice() {
val presented = assistantSnapshotForPresentation(
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
transcript = "private request",
response = "private response",
notice = AssistantSessionNotice.NoSpeech,
error = "private route detail",
),
locked = true,
)
assertNull(presented.transcript)
assertEquals("", presented.response)
assertNull(presented.error)
assertEquals(AssistantSessionNotice.NoSpeech, presented.notice)
}
@Test
fun unlockedPresentation_restoresConversationContent() {
val snapshot = AssistantSessionSnapshot(
phase = AssistantSessionPhase.Speaking,
transcript = "request",
response = "response",
)
assertEquals(snapshot, assistantSnapshotForPresentation(snapshot, locked = false))
}
@Test
fun liveKeyguardState_overridesLaunchFallbackInBothDirections() {
assertTrue(
assistantPresentationLocked(
currentKeyguardLocked = true,
fallbackLocked = false,
)
)
assertFalse(
assistantPresentationLocked(
currentKeyguardLocked = false,
fallbackLocked = true,
)
)
}
@Test
fun statusSnapshots_areFencedToTheCurrentActivation() {
assertTrue(assistantSnapshotMatchesActivation("activation-b", "activation-b"))
assertFalse(assistantSnapshotMatchesActivation("activation-b", "activation-a"))
assertFalse(assistantSnapshotMatchesActivation("activation-b", null))
assertFalse(assistantSnapshotMatchesActivation(null, "activation-b"))
}
@Test
fun voiceStateRetainsItsOwningActivationAcrossLaterRuntimeChanges() {
val activationA = VoiceUiState(
voiceMode = true,
state = VoiceState.Listening,
assistantActivationId = "activation-a",
)
val currentRuntimeActivation = "activation-b"
assertEquals("activation-a", activationA.assistantActivationId)
assertFalse(
assistantSnapshotMatchesActivation(
expectedActivationId = currentRuntimeActivation,
receivedActivationId = activationA.assistantActivationId,
)
)
}
@Test
fun terminalVoiceStateRetainsActivationForClosedPublication() {
val exited = com.hermesandroid.relay.viewmodel.voiceSessionExitState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Speaking,
assistantActivationId = "activation-a",
)
)
assertFalse(exited.voiceMode)
assertEquals("activation-a", exited.assistantActivationId)
assertEquals(
AssistantSessionPhase.Closed,
AssistantSessionProtocol.snapshotFromVoiceState(exited).phase,
)
}
@Test
fun subsequentOrdinaryVoiceEntryClearsPreviousAssistantOwner() {
val ordinaryEntry = VoiceUiState(
voiceMode = true,
state = VoiceState.Idle,
assistantActivationId = null,
)
assertNull(ordinaryEntry.assistantActivationId)
}
@Test
fun persistedSessionMarker_expiresAfterBoundedRecoveryWindow() {
val now = 2_000_000L
@@ -1,6 +1,9 @@
package com.hermesandroid.relay.viewmodel
import com.hermesandroid.relay.assistant.AssistantSessionNotice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
@@ -20,4 +23,35 @@ class VoiceCaptureGuardTest {
fun acceptsSettledShortUtterance() {
assertFalse(shouldDiscardVoiceCapture(durationMs = 420L, pcmBytes = 12_000))
}
@Test
fun noSpeechReturnsToRetryableIdleWithDurableFeedback() {
val state = voiceNoSpeechState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Transcribing,
transcribedText = "stale",
)
)
assertEquals(VoiceState.Idle, state.state)
assertEquals(AssistantSessionNotice.NoSpeech, state.assistantNotice)
assertNull(state.error)
assertNull(state.transcribedText)
assertTrue(state.voiceMode)
}
@Test
fun unrelatedCaptureCancellationDoesNotClaimNoSpeech() {
val state = voiceCaptureCancellationState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Listening,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
)
assertEquals(VoiceState.Idle, state.state)
assertNull(state.assistantNotice)
}
}
+5
View File
@@ -2495,6 +2495,8 @@ boundary.
- Activation heartbeats let the main runtime clean up after assistant-process loss.
Finish and show-failure paths clear pending/watchdog state, while Full Voice
explicitly transfers ownership before the session overlay stops heartbeats.
A recreated session process requests the current activation-fenced voice
snapshot rather than treating its empty local state as authoritative.
- Connection, chat, and voice runtime ownership is application-lifetime in the
main process rather than Activity-owned. The assistant service may initialize
that graph and start a turn while no Activity exists; the app UI later binds
@@ -2508,6 +2510,9 @@ boundary.
- Background and locked-screen invocation is mediated by Android's selected
assistant UI/session rather than an ordinary background Activity launch.
- Locked assistant UI exposes only generic phase and retry status. Transcript,
response, route-specific errors, diagnostics, and screen context remain hidden
until the device is unlocked; no-speech retry copy is deliberately content-free.
- Users can leave Hermes selected for gesture/power-button invocation while
turning continuous KWS off, or remove Hermes through Android's Assistant
settings.
+6 -1
View File
@@ -1015,7 +1015,10 @@ utilities.
application-lifetime owner, allowing assistant activation to start cold
without constructing or foregrounding `MainActivity`; full Voice later binds
that same runtime. Cancel, error, app/process recreation, and session finish
use the same scoped protocol. The
use the same scoped protocol. Recreated session UI requests an
activation-fenced snapshot from the app runtime, and capture/no-speech exits
retain a generic retry notice instead of collapsing to an unexplained Ready
state. The
wake recorder is released before the established voice recorder opens, and
assistant listening resumes only after the session exits. This mode is
mutually exclusive with the experimental notification-based foreground
@@ -1046,6 +1049,8 @@ utilities.
Realtime Agent sessions do not claim inclusion. The mic control follows the
active voice state, close remains separate, and **Open full voice** explicitly
transfers ownership so assistant-process cleanup cannot cancel the main-app flow.
While keyguard is active, the surface keeps only generic phase and retry copy;
transcript, response, route-specific errors, and screen context remain hidden.
- Stable voice integrates with `ChatViewModel` by **observing** `messages: StateFlow`; transcribed text goes through normal `chatVm.sendMessage(text)` so voice utterances appear as regular user messages in chat history. Experimental Realtime Agent creates a mirrored chat turn and applies broker events directly so tool state, transcript text, assistant deltas, and final responses appear without leaving voice mode.
- `VoiceModeOverlay` — full-screen UI with the MorphingSphere at 60% height in `voiceMode=true`, transcribed + response text, mic button supporting Tap / Hold / Continuous interaction modes.
- The optional `SYSTEM_ALERT_WINDOW` Voice control is user-invoked from an