Compare commits

..
Author SHA1 Message Date
Bailey Dixon 578c074797 fix(android): surface assistant capture recovery 2026-08-29 13:37:35 -04:00
20 changed files with 412 additions and 684 deletions
+1 -1
View File
@@ -17,8 +17,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
### Fixed
- **Opening Android no longer claims or interrupts a turn already running in Hermes Desktop/TUI.** Passive foreground and session browsing now use read-only Gateway status plus profile-scoped history; live-session resume remains reserved for explicit Android actions and exact Android-owned recovery.
- **The visible Android Sphere keeps its smooth procedural motion across startup and chat.** Backgrounded and motion-disabled surfaces remain still without reducing foreground animation to a stepped ambient pulse.
- **Android Assistant sessions explain when no speech was captured instead of appearing stuck at Ready.** Retry feedback survives the separate system overlay process, recreated session UI requests the current turn state, and locked sessions keep transcript, response, and technical error text private.
### Removed
+6 -4
View File
@@ -26,9 +26,6 @@ model device-certified:
renders as Working.
- Run a background process that outlives its parent turn and verify Background
work remains separate from the conversation's Idle state.
- On a physical phone, open and repeatedly foreground Android while the same
session is working in official Desktop/TUI; verify Android sends no live
attach/interrupt RPC, the producer completes, and final history appears.
- Pursue an upstream `session.active_list` profile field/filter or an aggregate
activity route with explicit profile ownership so multi-profile clients do
not need to resolve process-wide rows from durable keys.
@@ -1303,7 +1300,12 @@ and whether the agent is waiting on the user.
permissions; exercise compact, expanded, collapsed, and full-Voice handoff
states, background tap-through, rotation and insets, cancel/back, microphone
denial, network failure, process kill/recreation, and wake→voice→wake
resumption. Measure idle battery drain because third-party assistants do not
resumption. For background and keyguard capture, record `AudioRecord`, AppOps,
and foreground-service state: the user-installed app owns capture outside the
separate session process, so confirm whether the selected Assistant role is
sufficient on each target OS or whether activation needs an explicit,
activation-scoped microphone foreground-service lease. Measure idle battery
drain because third-party assistants do not
receive Google's dedicated low-power hotword hardware.
- **Audio quality guardrails** — normalize output volume across realtime and
@@ -239,60 +239,6 @@ class GatewayForegroundRecoveryInstrumentedTest {
assertEquals(0, fixture.requestsTo("/v1/chat/completions"))
}
@Test
fun desktopOwnedTurn_remainsReadOnlyAcrossAndroidForegroundLifecycle() {
viewModel.setChatVisible(false)
viewModel.updateGatewayClient(null)
gatewayClient.shutdown()
gatewayScope.cancel()
val controlMethods = setOf(
"session.resume",
"session.activate",
"session.interrupt",
"prompt.submit",
)
val baseline = controlMethods.associateWith(fixture::rpcCount)
val baselineActiveList = fixture.rpcCount("session.active_list")
fixture.activeSessionStatus = "working"
gatewayScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val okHttp = OkHttpClient()
gatewayClient = GatewayChatClient(
initialDashboardClient = DashboardApiClient(
baseUrl = fixture.server.url("/").toString().trimEnd('/'),
okHttpClient = okHttp,
),
okHttpClient = okHttp,
callbackDispatcher = { block -> Handler(Looper.getMainLooper()).post(block) },
scope = gatewayScope,
reconnectJitterUnit = { 0.0 },
)
viewModel.setChatTurnCheckpointStore(null)
viewModel.updateGatewayClient(gatewayClient)
viewModel.setChatVisible(true)
compose.activityRule.scenario.moveToState(Lifecycle.State.STARTED)
compose.activityRule.scenario.moveToState(Lifecycle.State.RESUMED)
viewModel.setChatVisible(false)
viewModel.setChatVisible(true)
fixture.awaitRpcCount("session.active_list", baselineActiveList + 1)
controlMethods.forEach { method ->
assertEquals(
"passive lifecycle sent $method",
baseline.getValue(method),
fixture.rpcCount(method),
)
}
viewModel.updateGatewayClient(null)
gatewayClient.shutdown()
assertEquals(
"observer teardown interrupted the Desktop turn",
baseline.getValue("session.interrupt"),
fixture.rpcCount("session.interrupt"),
)
}
private companion object {
const val STORED_SESSION_ID = "20260821_120000_fixture"
const val LIVE_SESSION_ID = "fixture-live-1"
@@ -317,9 +263,6 @@ internal class AndroidGatewayContractFixture {
@Volatile
var recoveryRunning = false
@Volatile
var activeSessionStatus: String? = null
private val listener = object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
sockets.add(webSocket)
@@ -339,18 +282,6 @@ internal class AndroidGatewayContractFixture {
"session.activate" -> sessionSnapshot(
(params["session_id"] as? JsonPrimitive)?.contentOrNull ?: "fixture-live-1",
)
"session.active_list" -> buildJsonObject {
put("sessions", kotlinx.serialization.json.buildJsonArray {
activeSessionStatus?.let { status ->
add(buildJsonObject {
put("id", LIVE_SESSION_ID)
put("session_key", STORED_SESSION_ID)
put("status", status)
put("last_active", 1.0)
})
}
})
}
"prompt.submit", "session.interrupt" -> buildJsonObject { put("ok", true) }
else -> JsonObject(emptyMap())
}
@@ -414,15 +345,6 @@ internal class AndroidGatewayContractFixture {
error("Gateway RPC $method not observed; saw ${rpcLog.map { it.first }}")
}
fun awaitRpcCount(method: String, count: Int) {
val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5)
while (System.nanoTime() < deadline) {
if (rpcCount(method) >= count) return
Thread.sleep(20)
}
error("Gateway RPC $method count $count not observed; saw ${rpcLog.map { it.first }}")
}
fun requestsTo(path: String): Int = requestPaths.count { it.startsWith(path) }
fun rpcCount(method: String): Int = rpcLog.count { it.first == method }
@@ -431,9 +353,4 @@ internal class AndroidGatewayContractFixture {
allSockets.forEach { socket -> runCatching { socket.close(1001, "teardown") } }
runCatching { server.shutdown() }
}
private companion object {
const val STORED_SESSION_ID = "20260821_120000_fixture"
const val LIVE_SESSION_ID = "fixture-live-1"
}
}
@@ -39,14 +39,38 @@ enum class AssistantSessionPhase {
Closed,
}
enum class AssistantSessionNotice {
NoSpeech,
}
data class AssistantSessionSnapshot(
val phase: AssistantSessionPhase = AssistantSessionPhase.Launching,
val transcript: String? = null,
val response: String = "",
val notice: AssistantSessionNotice? = null,
val error: String? = null,
val screenContextSupported: Boolean = false,
)
internal fun assistantSnapshotForPresentation(
snapshot: AssistantSessionSnapshot,
locked: Boolean,
): AssistantSessionSnapshot = if (locked) {
snapshot.copy(
transcript = null,
response = "",
error = null,
screenContextSupported = false,
)
} else {
snapshot
}
internal fun assistantSnapshotMatchesActivation(
expectedActivationId: String?,
receivedActivationId: String?,
): Boolean = expectedActivationId != null && expectedActivationId == receivedActivationId
object AssistantRole {
fun status(context: Context): AssistantRoleStatus {
val component = ComponentName(context, HermesVoiceInteractionService::class.java)
@@ -209,6 +233,7 @@ object AssistantSessionProtocol {
onFailure = { failure ->
publish(
application,
activation.id,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
@@ -219,13 +244,19 @@ object AssistantSessionProtocol {
return true
}
fun publish(context: Context, snapshot: AssistantSessionSnapshot) {
fun publish(
context: Context,
activationId: String,
snapshot: AssistantSessionSnapshot,
) {
context.sendBroadcast(
Intent(context, AssistantSessionStateReceiver::class.java).apply {
action = ACTION_STATUS
putExtra(EXTRA_ACTIVATION_ID, activationId)
putExtra(EXTRA_PHASE, snapshot.phase.name)
putExtra(EXTRA_TRANSCRIPT, snapshot.transcript)
putExtra(EXTRA_RESPONSE, snapshot.response)
putExtra(EXTRA_NOTICE, snapshot.notice?.name)
putExtra(EXTRA_ERROR, snapshot.error)
putExtra(EXTRA_SCREEN_CONTEXT_SUPPORTED, snapshot.screenContextSupported)
}
@@ -238,10 +269,6 @@ object AssistantSessionProtocol {
}
}
fun publish(context: Context, state: VoiceUiState) {
publish(context, snapshotFromVoiceState(state))
}
internal fun snapshotFromVoiceState(state: VoiceUiState): AssistantSessionSnapshot {
val phase = when {
!state.voiceMode -> AssistantSessionPhase.Closed
@@ -256,7 +283,10 @@ object AssistantSessionProtocol {
phase = phase,
transcript = state.transcribedText?.take(MAX_SESSION_TEXT_CHARS),
response = state.responseText.take(MAX_SESSION_TEXT_CHARS),
error = state.error?.take(MAX_SESSION_ERROR_CHARS),
notice = state.assistantNotice,
error = state.error
?.takeIf { phase == AssistantSessionPhase.Error }
?.take(MAX_SESSION_ERROR_CHARS),
)
}
@@ -350,6 +380,9 @@ object AssistantSessionProtocol {
phase = phase,
transcript = intent.getStringExtra(EXTRA_TRANSCRIPT),
response = intent.getStringExtra(EXTRA_RESPONSE).orEmpty(),
notice = intent.getStringExtra(EXTRA_NOTICE)?.let { raw ->
runCatching { AssistantSessionNotice.valueOf(raw) }.getOrNull()
},
error = intent.getStringExtra(EXTRA_ERROR),
screenContextSupported = intent.getBooleanExtra(
EXTRA_SCREEN_CONTEXT_SUPPORTED,
@@ -360,24 +393,33 @@ object AssistantSessionProtocol {
private const val MAX_SESSION_TEXT_CHARS = 4_000
private const val MAX_SESSION_ERROR_CHARS = 1_000
private const val EXTRA_NOTICE = "notice"
}
object AssistantSessionState {
private val _snapshot = MutableStateFlow(AssistantSessionSnapshot())
val snapshot: StateFlow<AssistantSessionSnapshot> = _snapshot.asStateFlow()
@Volatile private var activationId: String? = null
internal fun update(snapshot: AssistantSessionSnapshot) {
internal fun update(receivedActivationId: String?, snapshot: AssistantSessionSnapshot) {
if (!assistantSnapshotMatchesActivation(activationId, receivedActivationId)) return
_snapshot.value = snapshot
}
internal fun reset() {
internal fun reset(activationId: String) {
this.activationId = activationId
_snapshot.value = AssistantSessionSnapshot()
}
}
class AssistantSessionStateReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
AssistantSessionState.update(AssistantSessionProtocol.readSnapshot(intent))
AssistantSessionState.update(
receivedActivationId = intent.getStringExtra(
AssistantSessionProtocol.EXTRA_ACTIVATION_ID
),
snapshot = AssistantSessionProtocol.readSnapshot(intent),
)
}
}
@@ -423,6 +465,7 @@ class AssistantSessionLifecycleReceiver : BroadcastReceiver() {
onFailure = { failure ->
AssistantSessionProtocol.publish(
application,
id,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = failure.message ?: "Hermes voice could not start",
@@ -430,6 +473,7 @@ class AssistantSessionLifecycleReceiver : BroadcastReceiver() {
)
},
)
application.runtime.republishAssistantSnapshot(id)
return
}
if (AssistantSessionProtocol.isStartAction(intent.action)) {
@@ -3,6 +3,11 @@ package com.hermesandroid.relay.assistant
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.drawable.ColorDrawable
import android.app.KeyguardManager
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.os.Bundle
import android.service.voice.VoiceInteractionSession
import android.service.voice.VoiceInteractionSessionService
@@ -67,6 +72,7 @@ import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.LifecycleRegistry
@@ -108,6 +114,11 @@ internal fun shouldCancelVoiceWhenSessionUiEnds(
presentation: AssistantSessionPresentation,
): Boolean = presentation == AssistantSessionPresentation.Overlay
internal fun assistantPresentationLocked(
currentKeyguardLocked: Boolean?,
fallbackLocked: Boolean,
): Boolean = currentKeyguardLocked ?: fallbackLocked
private class HermesVoiceInteractionSession(
private val service: HermesVoiceInteractionSessionService,
) : VoiceInteractionSession(service) {
@@ -118,12 +129,19 @@ private class HermesVoiceInteractionSession(
private var surfaceExpanded by mutableStateOf(false)
private var activationId: String? = null
private var manualMic = false
private var keyguardLocked by mutableStateOf(false)
private var expectScreenContext: Boolean? = null
private var pendingSemantic = AssistantSemanticContext()
private var pendingScreenshot: ByteArray? = null
private var screenContextUi by mutableStateOf(AssistantScreenContextUi())
private val contextStore = assistantContextStore(service)
private var heartbeatJob: Job? = null
private var keyguardReceiverRegistered = false
private val keyguardReceiver = object : BroadcastReceiver() {
override fun onReceive(context: Context?, intent: Intent?) {
refreshKeyguardState()
}
}
init {
scope.launch {
@@ -139,6 +157,17 @@ private class HermesVoiceInteractionSession(
override fun onCreate() {
super.onCreate()
ContextCompat.registerReceiver(
service,
keyguardReceiver,
IntentFilter().apply {
addAction(Intent.ACTION_SCREEN_OFF)
addAction(Intent.ACTION_SCREEN_ON)
addAction(Intent.ACTION_USER_PRESENT)
},
ContextCompat.RECEIVER_NOT_EXPORTED,
)
keyguardReceiverRegistered = true
window.window?.apply {
setBackgroundDrawable(ColorDrawable(android.graphics.Color.TRANSPARENT))
clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND)
@@ -155,6 +184,7 @@ private class HermesVoiceInteractionSession(
PersistedHermesRelayTheme {
AssistantSessionSurface(
expanded = surfaceExpanded,
locked = keyguardLocked,
screenContext = screenContextUi,
onExpandedChange = { surfaceExpanded = it },
onCancel = { finishSession(cancelVoice = true) },
@@ -183,11 +213,22 @@ private class HermesVoiceInteractionSession(
override fun onShow(args: Bundle?, showFlags: Int) {
super.onShow(args, showFlags)
if (args?.getBoolean(HermesVoiceInteractionService.EXTRA_FROM_KEYGUARD, false) == true) {
refreshKeyguardState(
fallbackLocked = args?.getBoolean(
HermesVoiceInteractionService.EXTRA_FROM_KEYGUARD,
false,
) == true,
)
if (keyguardLocked) {
window.window?.addFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
} else {
window.window?.clearFlags(
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
)
}
setUiEnabled(true)
val startsNewLifecycle = presentation == AssistantSessionPresentation.Inactive
@@ -195,10 +236,10 @@ private class HermesVoiceInteractionSession(
if (!startsNewLifecycle) return
surfaceExpanded = false
AssistantSessionState.reset()
screenContextUi = AssistantScreenContextUi()
activationId = args?.getString(AssistantSessionProtocol.EXTRA_ACTIVATION_ID)
?: UUID.randomUUID().toString()
AssistantSessionState.reset(activationId!!)
manualMic = args?.getBoolean(AssistantSessionProtocol.EXTRA_MANUAL_MIC, false) ?: false
expectScreenContext = args?.getBoolean(
AssistantSessionProtocol.EXTRA_EXPECT_SCREEN_CONTEXT,
@@ -300,6 +341,10 @@ private class HermesVoiceInteractionSession(
pendingSemantic = AssistantSemanticContext()
pendingScreenshot = null
screenContextUi = AssistantScreenContextUi()
if (keyguardReceiverRegistered) {
runCatching { service.unregisterReceiver(keyguardReceiver) }
keyguardReceiverRegistered = false
}
viewOwner.stop()
scope.cancel()
super.onDestroy()
@@ -319,6 +364,7 @@ private class HermesVoiceInteractionSession(
)
}.onFailure {
AssistantSessionState.update(
activationId,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open the voice session.",
@@ -337,6 +383,7 @@ private class HermesVoiceInteractionSession(
setUiEnabled(false)
}.onFailure {
AssistantSessionState.update(
activationId,
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
error = it.message ?: "Hermes could not open full voice.",
@@ -377,6 +424,14 @@ private class HermesVoiceInteractionSession(
}
}
private fun refreshKeyguardState(fallbackLocked: Boolean = keyguardLocked) {
keyguardLocked = assistantPresentationLocked(
currentKeyguardLocked = service.getSystemService(KeyguardManager::class.java)
?.isKeyguardLocked,
fallbackLocked = fallbackLocked,
)
}
@RequiresApi(android.os.Build.VERSION_CODES.Q)
private fun stageAssistState(state: AssistState) {
stageAssistData(state.assistStructure, state.assistContent)
@@ -463,6 +518,7 @@ private class AssistantSessionViewOwner :
@Composable
private fun AssistantSessionSurface(
expanded: Boolean,
locked: Boolean,
screenContext: AssistantScreenContextUi,
onExpandedChange: (Boolean) -> Unit,
onCancel: () -> Unit,
@@ -471,7 +527,8 @@ private fun AssistantSessionSurface(
onOpenFullVoice: () -> Unit,
onSurfaceBoundsChanged: (android.graphics.Rect) -> Unit,
) {
val snapshot by AssistantSessionState.snapshot.collectAsState()
val rawSnapshot by AssistantSessionState.snapshot.collectAsState()
val snapshot = assistantSnapshotForPresentation(rawSnapshot, locked)
val status = assistantStatus(snapshot.phase)
val transmittedScreenContext = if (snapshot.screenContextSupported) {
screenContext
@@ -650,6 +707,13 @@ private fun ExpandedAssistantSurface(
color = MaterialTheme.colorScheme.onSurface,
)
}
snapshot.notice?.let { notice ->
Text(
text = assistantNoticeText(notice),
color = MaterialTheme.colorScheme.onSurfaceVariant,
style = MaterialTheme.typography.bodyMedium,
)
}
snapshot.error?.let { error ->
Text(
text = error,
@@ -896,5 +960,11 @@ private fun assistantStatus(phase: AssistantSessionPhase): String = when (phase)
private fun compactAssistantText(snapshot: AssistantSessionSnapshot): String =
snapshot.transcript?.takeIf { it.isNotBlank() }
?: snapshot.response.takeIf { it.isNotBlank() }
?: snapshot.notice?.let { assistantNoticeText(it) }
?: snapshot.error?.takeIf { it.isNotBlank() }
?: assistantStatus(snapshot.phase)
@Composable
private fun assistantNoticeText(notice: AssistantSessionNotice): String = when (notice) {
AssistantSessionNotice.NoSpeech -> stringResource(R.string.voice_no_speech_try_again)
}
@@ -906,30 +906,6 @@ class GatewayChatClient(
scope.launch { prewarmAwait(storedSessionId) }
}
/**
* Establish only the shared Gateway socket for read-only observation.
*
* `session.resume` and `session.activate` attach a live runtime to this
* transport. Opening Chat, foreground restoration, and selecting a saved
* transcript must not claim a turn that another Desktop/TUI client owns,
* so those paths use this socket-only warmup and observe through REST
* history plus `session.active_list` instead.
*/
fun observe(onReady: (() -> Unit)? = null) {
scope.launch {
if (observeAwait() && onReady != null) callbackDispatcher(onReady)
}
}
/** Suspending [observe]; returns true once the read-only socket is ready. */
suspend fun observeAwait(): Boolean = try {
connectMutex.withLock { ensureConnected() }
true
} catch (e: Exception) {
Log.d(TAG, "Gateway observation warmup skipped: ${e.message}")
false
}
/**
* Suspending [prewarm]: establishes the socket and (when [storedSessionId]
* is non-null) resumes the existing session, returning only once that work
@@ -240,6 +240,25 @@ class HermesProcessRuntime internal constructor(
}
}
fun republishAssistantSnapshot(activationId: String) {
val snapshot = synchronized(activationLock) {
if (currentActivationId != activationId ||
_initializationState.value != HermesRuntimeInitializationState.Ready
) {
null
} else {
binder.assistantSnapshot.value
}
} ?: return
if (snapshot.phase != com.hermesandroid.relay.assistant.AssistantSessionPhase.Closed) {
com.hermesandroid.relay.assistant.AssistantSessionProtocol.publish(
application,
activationId,
snapshot,
)
}
}
fun recordAssistantHeartbeat(
activationId: String,
nowElapsedMs: Long = SystemClock.elapsedRealtime(),
@@ -376,7 +376,9 @@ internal class HermesRuntimeBinder(
if (!AssistantAppSessionState.active.value) return@collect
if (state.voiceMode) AssistantAppSessionState.markVoiceStarted()
if (state.voiceMode || AssistantAppSessionState.hasVoiceStarted()) {
AssistantSessionProtocol.publish(application, snapshot)
state.assistantActivationId?.let { activationId ->
AssistantSessionProtocol.publish(application, activationId, snapshot)
}
}
}
}
@@ -1119,16 +1119,12 @@ fun ChatScreen(
}
// Recover any durable in-flight chat checkpoint whenever Chat returns to
// the foreground. setChatVisible owns that edge; an ordinary Gateway open
// warms only the observation socket and never attaches a saved session.
// the foreground. On Gateway this also pre-warms/re-attaches the socket;
// sessions-SSE falls back to bounded persisted-history reconciliation.
val appForeground by com.hermesandroid.relay.util.AppForegroundTracker.isForeground.collectAsState()
LaunchedEffect(isGatewayTransport, appForeground, chatReady) {
val chatVisible = appForeground && chatReady
val visibilityChanged = chatViewModel.setChatVisible(chatVisible)
if (isGatewayTransport && chatVisible && !visibilityChanged) {
// Gateway availability can settle after Chat was already visible.
// Repeat the socket-only warmup for that edge; ordinary observation
// still cannot resume or activate a session.
chatViewModel.setChatVisible(appForeground && chatReady)
if (appForeground && chatReady) {
chatViewModel.prewarmGateway()
}
if (isGatewayTransport && appForeground && chatReady) {
@@ -408,9 +408,6 @@ class ChatViewModel : ViewModel() {
private val sessionActivityGeneration = AtomicLong(0L)
private val sessionActivityPollMutex = Mutex()
private var sessionActivityPollJob: Job? = null
private var passiveGatewayHistoryRefreshJob: Job? = null
private var passivelyObservedGatewaySessionId: String? = null
private var passiveObservationCatchupPendingSessionId: String? = null
private var sessionActivityDirectory: Set<SessionActivityOwner> = emptySet()
private var lastProjectedProcessIds: Set<String> = emptySet()
private var lastProjectedProcessOwner: SessionActivityOwner? = null
@@ -2255,8 +2252,6 @@ class ChatViewModel : ViewModel() {
}
private suspend fun pollSessionActivity(client: GatewayChatClient) {
var hasPassiveCurrentLiveWork = false
var hasPassiveCatchupPending = false
sessionActivityPollMutex.withLock {
if (gatewayClient !== client || !chatVisible || streamingEndpoint != "gateway") return
val generation = sessionActivityGeneration.get()
@@ -2277,43 +2272,6 @@ class ChatViewModel : ViewModel() {
when (val result = client.listActiveSessions()) {
is GatewayActiveSessionsResult.Success -> {
if (gatewayClient !== client || generation != sessionActivityGeneration.get()) return
val currentStoredId = currentOwner?.storedSessionId
val passiveCurrentRows = if (currentStoredId == null) {
emptyList()
} else {
result.sessions.filter { row ->
row.storedSessionId == currentStoredId &&
client.knownSessionOwner(row.runtimeSessionId) == null
}
}
hasPassiveCurrentLiveWork = passiveCurrentRows.any { row ->
row.status != GatewayActiveSessionStatus.Idle
}
val initialCatchupPending =
passiveObservationCatchupPendingSessionId == currentStoredId
val needsFinalPassiveRefresh =
passivelyObservedGatewaySessionId == currentStoredId &&
!hasPassiveCurrentLiveWork
if (hasPassiveCurrentLiveWork) {
currentStoredId?.let(::refreshPassivelyObservedGatewayHistory)
if (initialCatchupPending) {
passiveObservationCatchupPendingSessionId = null
}
} else if (needsFinalPassiveRefresh || initialCatchupPending) {
val scheduled = currentStoredId?.let { storedId ->
refreshPassivelyObservedGatewayHistory(
storedSessionId = storedId,
retryUntilChanged = true,
)
} == true
if (scheduled && initialCatchupPending) {
passiveObservationCatchupPendingSessionId = null
}
}
passivelyObservedGatewaySessionId =
currentStoredId?.takeIf { hasPassiveCurrentLiveWork }
hasPassiveCatchupPending =
passiveObservationCatchupPendingSessionId == currentStoredId
val resolved = resolveGatewayActiveSessions(
sessions = result.sessions,
directory = directory,
@@ -2375,18 +2333,6 @@ class ChatViewModel : ViewModel() {
GatewayActiveSessionsResult.Unsupported,
is GatewayActiveSessionsResult.TransientFailure -> {
if (gatewayClient !== client || generation != sessionActivityGeneration.get()) return
val currentStoredId = currentOwner?.storedSessionId
if (passiveObservationCatchupPendingSessionId == currentStoredId) {
val scheduled = currentStoredId?.let { storedId ->
refreshPassivelyObservedGatewayHistory(
storedSessionId = storedId,
retryUntilChanged = true,
)
} == true
if (scheduled) passiveObservationCatchupPendingSessionId = null
}
hasPassiveCatchupPending =
passiveObservationCatchupPendingSessionId == currentStoredId
val scopes = directory.mapTo(mutableSetOf()) {
SessionActivityScope.of(it.connectionId, it.profile)
}.apply { add(currentScope) }
@@ -2405,9 +2351,7 @@ class ChatViewModel : ViewModel() {
record.freshness == SessionActivityFreshness.Confirmed &&
record.phase(System.currentTimeMillis()) != SessionActivityPhase.Idle
}
val delayMs = if (
hasConfirmedLiveWork || hasPassiveCurrentLiveWork || hasPassiveCatchupPending
) 1_500L else 30_000L
val delayMs = if (hasConfirmedLiveWork) 1_500L else 30_000L
sessionActivityPollJob = viewModelScope.launch {
delay(delayMs)
if (gatewayClient === client && chatVisible) pollSessionActivity(client)
@@ -2480,10 +2424,6 @@ class ChatViewModel : ViewModel() {
clearProjectedBackgroundProcesses()
sessionActivityPollJob?.cancel()
sessionActivityPollJob = null
passiveGatewayHistoryRefreshJob?.cancel()
passiveGatewayHistoryRefreshJob = null
passivelyObservedGatewaySessionId = null
passiveObservationCatchupPendingSessionId = null
sessionActivityGeneration.incrementAndGet()
sessionActivityDirectory = emptySet()
lastLocalActivityOwner = null
@@ -2603,9 +2543,8 @@ class ChatViewModel : ViewModel() {
// Foreground can race OkHttp's delayed close callback:
// the first prewarm sees the old socket as Ready, then
// the callback moves it to Idle. Re-run from this exact
// client transition so the observation socket is
// restored; only an exact Android checkpoint may
// resume/activate a live runtime.
// client transition so the visible durable session is
// resumed and its authoritative history reconciled.
prewarmGateway()
}
}
@@ -3060,103 +2999,6 @@ class ChatViewModel : ViewModel() {
}
}
/**
* Refresh a Desktop/TUI-owned turn through the profile-scoped history
* surface without attaching its live runtime. `session.active_list` drives
* the bounded cadence; one final read follows Working/Waiting -> Idle.
*/
private fun refreshPassivelyObservedGatewayHistory(
storedSessionId: String,
retryUntilChanged: Boolean = false,
): Boolean {
if (passiveGatewayHistoryRefreshJob?.isActive == true) return false
if (_isLoadingHistory.value) return false
val handler = chatHandler ?: return false
val contextKey = activeProfileContextKey
val profileName = currentSessionProfileName()
val refreshJob = viewModelScope.launch(start = CoroutineStart.LAZY) {
try {
repeat(if (retryUntilChanged) 8 else 1) { attempt ->
val serverMessages = runCatching {
loadGatewaySessionHistory(
sessionId = storedSessionId,
requireProfileScope = true,
profileName = profileName,
)
}.getOrNull() ?: return@launch
if (
chatHandler !== handler ||
activeProfileContextKey != contextKey ||
currentSessionProfileName() != profileName ||
handler.currentSessionId.value != storedSessionId ||
_isLoadingHistory.value ||
activeStream != null ||
handler.isStreaming.value
) return@launch
val visibleSignature = handler.messages.value
.filterNot { it.clientOnly }
.map { message ->
Triple(
message.role.name.lowercase(),
message.content,
message.thinkingContent,
)
}
val serverSignature = serverMessages.map { message ->
Triple(
message.role.lowercase(),
message.contentText.orEmpty(),
message.resolvedReasoning.orEmpty(),
)
}
if (visibleSignature != serverSignature) {
handler.loadMessageHistory(serverMessages)
refreshSessions()
scheduleTitleReconcile(storedSessionId)
return@launch
}
if (attempt < 7 && retryUntilChanged) delay(250L)
}
} finally {
if (passiveGatewayHistoryRefreshJob === coroutineContext[Job]) {
passiveGatewayHistoryRefreshJob = null
}
}
}
passiveGatewayHistoryRefreshJob = refreshJob
refreshJob.start()
return true
}
/** Open the read-only socket off Main, then publish observation ownership on Main. */
private fun observeGatewaySession(
client: GatewayChatClient?,
handler: ChatHandler,
storedSessionId: String,
) {
val observer = client ?: return
val contextKey = activeProfileContextKey
val profileName = currentSessionProfileName()
observer.observe {
if (
chatVisible &&
gatewayClient === observer &&
chatHandler === handler &&
activeProfileContextKey == contextKey &&
currentSessionProfileName() == profileName &&
handler.currentSessionId.value == storedSessionId
) {
passiveObservationCatchupPendingSessionId = storedSessionId
refreshPassivelyObservedGatewayHistory(
storedSessionId = storedSessionId,
retryUntilChanged = true,
)
requestSessionActivityRefresh()
}
}
}
/** One-shot `config.get personality` over a ready socket → drives the collector. */
private fun seedServerPersonality(client: GatewayChatClient) {
viewModelScope.launch {
@@ -3167,11 +3009,11 @@ class ChatViewModel : ViewModel() {
}
/**
* Warm the Gateway socket when Chat is visible without claiming a runtime
* that may belong to Desktop/TUI. Exact Android-owned checkpoints recover
* through `session.activate`/`session.resume`; an ordinary open observes
* through REST history and `session.active_list` until the user performs
* an explicit action that needs session ownership.
* Warm the gateway socket (and resume the current session) when the chat
* surface is visible and the gateway is the resolved transport, so the
* first send is warm instead of paying the cold connect + `session.resume`
* on the send path. No-op without a gateway client; idempotent when warm.
* Driven by a foreground/visibility effect in ChatScreen.
*/
fun prewarmGateway() {
val client = gatewayClient
@@ -3179,16 +3021,17 @@ class ChatViewModel : ViewModel() {
val sessionId = handler.currentSessionId.value
selectBackgroundProcessSession(sessionId)
if (sessionId == null) {
client?.observe()
client?.prewarm(null)
} else {
// Preserve the original warm-up path before persistence wiring is
// available (early composition and JVM tests). Production installs
// the store from initializeMedia before Chat becomes ready.
if (chatTurnCheckpointStore == null) {
val gateway = client ?: return
// GatewayChatClient owns the socket IO scope, so the dial can
// progress while a paused UI dispatcher is being recreated.
observeGatewaySession(gateway, handler, sessionId)
// GatewayChatClient owns an IO scope, so this can progress even
// while a paused/blocked UI dispatcher is being recreated.
// Its cold-ready listener performs history/process refresh.
gateway.prewarm(sessionId)
return
}
if (activeStream == null && (streamRecovery == null || client != null)) {
@@ -3200,29 +3043,26 @@ class ChatViewModel : ViewModel() {
chatHandler === handler &&
handler.currentSessionId.value == sessionId
) {
observeGatewaySession(client, handler, sessionId)
if (client?.prewarmAwait(sessionId) == true) {
gatewayProcessController.sessionReady(sessionId)
}
}
checkpointRecoveryJob = null
}
return
}
// A locally-owned live mapper may revalidate its existing binding.
// A passive transcript must remain socket-only: resuming it here
// can replace another client's transport and turn Android teardown
// into a later session.interrupt.
if (client?.hasActiveTurnForSession(sessionId) == true) {
viewModelScope.launch {
if (client.prewarmAwait(sessionId) &&
gatewayClient === client &&
chatHandler === handler &&
handler.currentSessionId.value == sessionId
) {
gatewayProcessController.sessionReady(sessionId)
requestSessionActivityRefresh()
}
// prewarm() only emits the existing "cold ready" callback when it
// had to resume. An already-live session still needs its initial
// process snapshot when Chat opens, so confirm it explicitly.
viewModelScope.launch {
if (
client?.prewarmAwait(sessionId) == true &&
gatewayClient === client &&
chatHandler === handler &&
handler.currentSessionId.value == sessionId
) {
gatewayProcessController.sessionReady(sessionId)
}
} else {
observeGatewaySession(client, handler, sessionId)
}
}
}
@@ -3232,7 +3072,7 @@ class ChatViewModel : ViewModel() {
* Gateway chat owns automatic idle-socket reattachment; other tabs and a
* backgrounded app retain the normal no-reconnect behavior.
*/
fun setChatVisible(visible: Boolean): Boolean {
fun setChatVisible(visible: Boolean) {
val changed = chatVisible != visible
chatVisible = visible
if (visible && changed) {
@@ -3241,12 +3081,7 @@ class ChatViewModel : ViewModel() {
} else if (!visible) {
sessionActivityPollJob?.cancel()
sessionActivityPollJob = null
passiveGatewayHistoryRefreshJob?.cancel()
passiveGatewayHistoryRefreshJob = null
passivelyObservedGatewaySessionId = null
passiveObservationCatchupPendingSessionId = null
}
return changed
}
// === Gateway desktop-parity state ===
@@ -4630,9 +4465,7 @@ class ChatViewModel : ViewModel() {
)
if (stillCurrent()) {
handler.loadMessageHistory(messages)
if (streamingEndpoint == "gateway") {
observeGatewaySession(gatewayClient, handler, sessionId)
}
if (streamingEndpoint == "gateway") gatewayClient?.prewarm(sessionId)
}
}
} catch (e: kotlinx.coroutines.CancellationException) {
@@ -5143,9 +4976,7 @@ class ChatViewModel : ViewModel() {
handler.currentSessionId.value == sessionId
) {
handler.loadMessageHistory(messages)
if (streamingEndpoint == "gateway") {
observeGatewaySession(gatewayClient, handler, sessionId)
}
if (streamingEndpoint == "gateway") gatewayClient?.prewarm(sessionId)
}
} catch (e: kotlinx.coroutines.CancellationException) {
throw e
@@ -6735,10 +6566,6 @@ class ChatViewModel : ViewModel() {
* SSE cannot, so it retains the existing interrupt/cancel behavior.
*/
private fun releaseTurnForNavigation(handler: ChatHandler) {
passiveGatewayHistoryRefreshJob?.cancel()
passiveGatewayHistoryRefreshJob = null
passivelyObservedGatewaySessionId = null
passiveObservationCatchupPendingSessionId = null
val gateway = gatewayClient
val canBackground = streamingEndpoint == "gateway" &&
activeStreamIsGateway && activeStream != null && gateway != null
@@ -52,6 +52,7 @@ import com.hermesandroid.relay.voice.VoiceCommandInterpreter
import com.hermesandroid.relay.voice.SpokenInterruptionLatch
import com.hermesandroid.relay.voice.voiceInterfaceContextPrompt
import com.hermesandroid.relay.assistant.assistantContextStore
import com.hermesandroid.relay.assistant.AssistantSessionNotice
import com.hermesandroid.relay.assistant.buildAssistantVoiceTurnPayload
// === PHASE3-voice-intents: voice→bridge intent routing ===
import com.hermesandroid.relay.voice.IntentResult
@@ -126,6 +127,25 @@ internal fun voiceSubmissionRetryState(state: VoiceUiState): VoiceUiState = stat
error = null,
)
internal fun voiceNoSpeechState(state: VoiceUiState): VoiceUiState = state.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
transcribedText = null,
error = null,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
internal fun voiceCaptureCancellationState(
state: VoiceUiState,
notice: AssistantSessionNotice? = null,
): VoiceUiState = state.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
assistantNotice = notice,
)
internal data class AssistantContextTurnDisposition(
val retireForLaterTurns: Boolean,
val consumeOnTransportAcceptance: Boolean,
@@ -328,6 +348,10 @@ data class VoiceUiState(
val responseText: String = "",
/** Human-readable error surfaced in the overlay. */
val error: String? = null,
/** Content-free retry status safe for the system Assistant surface. */
val assistantNotice: AssistantSessionNotice? = null,
/** Stable owner for cross-process Assistant status; null for ordinary voice. */
val assistantActivationId: String? = null,
/** Currently-selected interaction mode. */
val interactionMode: InteractionMode = InteractionMode.TapToTalk,
/**
@@ -440,6 +464,7 @@ internal fun voiceSessionExitState(state: VoiceUiState): VoiceUiState =
transcribedText = null,
responseText = "",
error = null,
assistantNotice = null,
destructiveCountdown = null,
hermesConfirmation = null,
handoffStatus = null,
@@ -1630,6 +1655,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
state = VoiceState.Idle,
outputAudioActive = false,
error = null,
assistantActivationId = activationId,
hermesConfirmation = null,
backgroundRun = if (orphanedRun != null) null else it.backgroundRun,
)
@@ -2067,6 +2093,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
state = VoiceState.Listening,
outputAudioActive = false,
error = null,
assistantNotice = null,
responseText = "",
// v0.4.1 — fresh turn, drop any stale JIT permission chip
// from the previous dispatch.
@@ -2176,7 +2203,11 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
private fun shouldDiscardVoiceCaptureBeforeStop(durationMs: Long): Boolean =
durationMs < MIN_VOICE_CAPTURE_DURATION_MS
private fun cancelListeningWithoutProcessing(title: String, detail: String? = null) {
private fun cancelListeningWithoutProcessing(
title: String,
detail: String? = null,
notice: AssistantSessionNotice? = null,
) {
responseInterruptedForVoiceCommand = false
silenceWatchdogJob?.cancel()
silenceWatchdogJob = null
@@ -2188,9 +2219,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
title = title,
detail = detail,
)
_uiState.update {
it.copy(state = VoiceState.Idle, amplitude = 0f, outputAudioActive = false)
}
_uiState.update { voiceCaptureCancellationState(it, notice) }
}
/** Reconcile microphone state after the Activity returns to foreground. */
@@ -2327,6 +2356,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
cancelListeningWithoutProcessing(
title = getApplication<Application>().getString(R.string.voice_status_no_speech),
detail = "No speech within ${IDLE_NO_SPEECH_MS / 1000}s",
notice = AssistantSessionNotice.NoSpeech,
)
return@launch
}
@@ -6497,13 +6527,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
detail = detail,
)
_uiState.update {
it.copy(
state = VoiceState.Idle,
amplitude = 0f,
outputAudioActive = false,
error = null,
transcribedText = null,
)
voiceNoSpeechState(it)
}
Toast.makeText(context, message, Toast.LENGTH_SHORT).show()
}
@@ -65,6 +65,122 @@ class AssistantSessionProtocolTest {
assertEquals("Microphone unavailable", error.error)
}
@Test
fun idleNoSpeech_isAVisibleRetryNotice() {
val snapshot = AssistantSessionProtocol.snapshotFromVoiceState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Idle,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
)
assertEquals(AssistantSessionPhase.Idle, snapshot.phase)
assertEquals(AssistantSessionNotice.NoSpeech, snapshot.notice)
assertNull(snapshot.error)
}
@Test
fun lockedPresentation_redactsConversationButKeepsGenericNotice() {
val presented = assistantSnapshotForPresentation(
AssistantSessionSnapshot(
phase = AssistantSessionPhase.Error,
transcript = "private request",
response = "private response",
notice = AssistantSessionNotice.NoSpeech,
error = "private route detail",
),
locked = true,
)
assertNull(presented.transcript)
assertEquals("", presented.response)
assertNull(presented.error)
assertEquals(AssistantSessionNotice.NoSpeech, presented.notice)
}
@Test
fun unlockedPresentation_restoresConversationContent() {
val snapshot = AssistantSessionSnapshot(
phase = AssistantSessionPhase.Speaking,
transcript = "request",
response = "response",
)
assertEquals(snapshot, assistantSnapshotForPresentation(snapshot, locked = false))
}
@Test
fun liveKeyguardState_overridesLaunchFallbackInBothDirections() {
assertTrue(
assistantPresentationLocked(
currentKeyguardLocked = true,
fallbackLocked = false,
)
)
assertFalse(
assistantPresentationLocked(
currentKeyguardLocked = false,
fallbackLocked = true,
)
)
}
@Test
fun statusSnapshots_areFencedToTheCurrentActivation() {
assertTrue(assistantSnapshotMatchesActivation("activation-b", "activation-b"))
assertFalse(assistantSnapshotMatchesActivation("activation-b", "activation-a"))
assertFalse(assistantSnapshotMatchesActivation("activation-b", null))
assertFalse(assistantSnapshotMatchesActivation(null, "activation-b"))
}
@Test
fun voiceStateRetainsItsOwningActivationAcrossLaterRuntimeChanges() {
val activationA = VoiceUiState(
voiceMode = true,
state = VoiceState.Listening,
assistantActivationId = "activation-a",
)
val currentRuntimeActivation = "activation-b"
assertEquals("activation-a", activationA.assistantActivationId)
assertFalse(
assistantSnapshotMatchesActivation(
expectedActivationId = currentRuntimeActivation,
receivedActivationId = activationA.assistantActivationId,
)
)
}
@Test
fun terminalVoiceStateRetainsActivationForClosedPublication() {
val exited = com.hermesandroid.relay.viewmodel.voiceSessionExitState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Speaking,
assistantActivationId = "activation-a",
)
)
assertFalse(exited.voiceMode)
assertEquals("activation-a", exited.assistantActivationId)
assertEquals(
AssistantSessionPhase.Closed,
AssistantSessionProtocol.snapshotFromVoiceState(exited).phase,
)
}
@Test
fun subsequentOrdinaryVoiceEntryClearsPreviousAssistantOwner() {
val ordinaryEntry = VoiceUiState(
voiceMode = true,
state = VoiceState.Idle,
assistantActivationId = null,
)
assertNull(ordinaryEntry.assistantActivationId)
}
@Test
fun persistedSessionMarker_expiresAfterBoundedRecoveryWindow() {
val now = 2_000_000L
@@ -231,14 +231,11 @@ class GatewayClientHarness(
val suppressAckMethods: MutableSet<String> = ConcurrentHashMap.newKeySet()
val pendingAcks = LinkedBlockingQueue<PendingAck>()
@Volatile
var suppressGatewayReady: Boolean = false
private val wsListener = object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: okhttp3.Response) {
serverSockets.add(webSocket)
allServerSockets.add(webSocket)
if (!suppressGatewayReady) sendGatewayReady(webSocket)
webSocket.send(eventFrame("gateway.ready", null, null))
}
override fun onMessage(webSocket: WebSocket, text: String) {
@@ -630,10 +627,6 @@ class GatewayClientHarness(
fun awaitServerSocket(): WebSocket =
serverSockets.poll(5, TimeUnit.SECONDS) ?: error("server socket never opened")
fun sendGatewayReady(webSocket: WebSocket) {
webSocket.send(eventFrame("gateway.ready", null, null))
}
fun awaitRpc(method: String): JsonObject {
val deadline = System.currentTimeMillis() + 5_000
while (System.currentTimeMillis() < deadline) {
@@ -1414,27 +1407,6 @@ class GatewayChatClientTest {
assertEquals(listOf("stored-session"), resumedSessions.toList())
}
@Test
fun `observation warmup never claims or interrupts a foreign runtime`() = runBlocking {
val registrations = AtomicInteger(0)
client.setUnsolicitedTurnProvider {
registrations.incrementAndGet()
GatewayInboundTurnRegistration(Recorder().callbacks) { true }
}
assertTrue(client.observeAwait())
val serverWs = harness.awaitServerSocket()
serverWs.send(harness.eventFrame("message.start", null, "foreign-runtime"))
delay(100)
client.shutdown()
assertEquals(0, registrations.get())
assertFalse(harness.rpcLog.any { it.first == "session.resume" })
assertFalse(harness.rpcLog.any { it.first == "session.activate" })
assertFalse(harness.rpcLog.any { it.first == "session.interrupt" })
assertFalse(harness.rpcLog.any { it.first == "prompt.submit" })
}
@Test
fun `newer prewarm selection wins when an older resume completes late`() = runBlocking {
harness.suppressAckMethods += "session.resume"
@@ -992,12 +992,11 @@ class ChatViewModelGatewayInboundTurnTest {
awaitCondition { handler.messages.value.any { it.content == "Partial A" } }
viewModel.switchSession(secondSession)
gatewayHarness.awaitRpcCount("session.resume", 2)
awaitCondition { handler.currentSessionId.value == secondSession && !handler.isStreaming.value }
assertEquals(1, gatewayHarness.rpcLog.count { it.first == "session.resume" })
assertTrue(gatewayHarness.rpcLog.none { it.first == "session.interrupt" })
viewModel.sendMessage("Run task B")
gatewayHarness.awaitRpcCount("session.resume", 2)
gatewayHarness.awaitRpcCount("prompt.submit", 2)
serverWs.send(
gatewayHarness.eventFrame(
@@ -1825,12 +1824,11 @@ class ChatViewModelGatewayInboundTurnTest {
awaitCondition { viewModel.queuedMessages.value == listOf("Follow up A") }
viewModel.switchSession(secondSession)
gatewayHarness.awaitRpcCount("session.resume", 2)
awaitCondition { handler.currentSessionId.value == secondSession && !handler.isStreaming.value }
assertEquals(1, gatewayHarness.rpcLog.count { it.first == "session.resume" })
assertTrue("session B must not show A's queue", viewModel.queuedMessages.value.isEmpty())
viewModel.sendMessage("Run task B")
gatewayHarness.awaitRpcCount("session.resume", 2)
gatewayHarness.awaitRpcCount("prompt.submit", 2)
serverWs.send(
gatewayHarness.eventFrame(
@@ -2033,16 +2031,13 @@ class ChatViewModelGatewayInboundTurnTest {
serverWs.close(1012, "test disconnect")
awaitCondition { gatewayClient.connectionState.value == GatewayConnectionState.Idle }
viewModel.setChatVisible(true)
viewModel.prewarmGateway()
gatewayHarness.awaitServerSocket()
gatewayHarness.awaitRpcCount("session.resume", 2)
awaitCondition {
handler.messages.value.singleOrNull()?.content == BACKGROUND_ANSWER
}
assertEquals(1, gatewayHarness.rpcLog.count { it.first == "session.resume" })
assertEquals(0, gatewayHarness.rpcLog.count { it.first == "session.activate" })
assertEquals(0, gatewayHarness.rpcLog.count { it.first == "session.interrupt" })
assertFalse(handler.isStreaming.value)
}
@@ -2053,19 +2048,16 @@ class ChatViewModelGatewayInboundTurnTest {
// Foreground arrives while OkHttp still reports the old socket ready,
// so the one-shot prewarm is an intentional no-op. The delayed close
// callback must itself restore the observation socket and catch up
// history without attaching the live session.
// callback must itself trigger an exact-session reattach.
viewModel.prewarmGateway()
serverWs.close(1012, "late background close")
awaitCondition { gatewayHarness.ticketMints.get() >= 2 }
serverWs = gatewayHarness.awaitServerSocket()
gatewayHarness.awaitRpcCount("session.resume", 2)
awaitCondition {
handler.messages.value.singleOrNull()?.content == BACKGROUND_ANSWER
}
assertEquals(1, gatewayHarness.rpcLog.count { it.first == "session.resume" })
assertEquals(0, gatewayHarness.rpcLog.count { it.first == "session.activate" })
assertEquals(0, gatewayHarness.rpcLog.count { it.first == "session.interrupt" })
assertFalse(handler.isStreaming.value)
}
@@ -2374,189 +2366,36 @@ class ChatViewModelGatewayInboundTurnTest {
}
@Test
fun reconnectCatchupClosesCompletionBetweenFirstReadAndIdleSnapshot() {
viewModel.switchProfileContext(PROFILE_CONTEXT, STORED_SESSION_ID)
awaitCondition { !viewModel.isLoadingHistory.value }
val firstReadStarted = CompletableDeferred<Unit>()
val releaseFirstRead = CompletableDeferred<Unit>()
val readCount = AtomicInteger(0)
viewModel.setProfileMessageLoader {
if (readCount.incrementAndGet() == 1) {
firstReadStarted.complete(Unit)
releaseFirstRead.await()
Result.success(emptyList())
} else {
Result.success(persistedHistory)
}
}
fun reconnectAfterMissedStartRecoversOnExactSessionCompletion() {
serverWs.close(1012, "missed start")
awaitCondition { gatewayClient.connectionState.value == GatewayConnectionState.Idle }
viewModel.setChatVisible(true)
viewModel.prewarmGateway()
serverWs = gatewayHarness.awaitServerSocket()
awaitCondition { firstReadStarted.isCompleted }
// Completion persists after the reconnect's first catch-up read began,
// while the first active-list snapshot is already empty/idle. The
// pending final-read marker must close this exact ordering window.
gatewayHarness.awaitRpcCount("session.resume", 2)
// Reconnected midway through the synthetic turn: no message.start is
// replayed, so the delta is intentionally ignored and completion drives
// authoritative history recovery.
serverWs.send(
gatewayHarness.eventFrame(
"message.delta",
buildJsonObject { put("text", BACKGROUND_ANSWER) },
"live-resumed",
),
)
persistedHistory = persistedAnswerHistory()
releaseFirstRead.complete(Unit)
serverWs.send(
gatewayHarness.eventFrame(
"message.complete",
buildJsonObject { put("text", BACKGROUND_ANSWER) },
"live-resumed",
),
)
awaitCondition { handler.messages.value.any { it.content == BACKGROUND_ANSWER } }
assertEquals(1, gatewayHarness.rpcLog.count { it.first == "session.resume" })
assertEquals(0, gatewayHarness.rpcLog.count { it.first == "session.activate" })
assertEquals(0, gatewayHarness.rpcLog.count { it.first == "session.interrupt" })
assertFalse(handler.isStreaming.value)
}
@Test
fun passiveForegroundObservationNeverClaimsOrInterruptsDesktopTurn() {
val observerProfile = Profile(
name = "observer",
model = "model-a",
description = "Observer",
)
viewModel.setSelectedProfileProvider { observerProfile }
viewModel.setSessionProfileNameProvider { observerProfile.name }
viewModel.setProfileMessageLoaderWithMode { profileName, sessionId, _ ->
assertEquals(STORED_SESSION_ID, sessionId)
Result.success(
if (profileName == observerProfile.name) {
persistedHistory
} else {
listOf(
MessageItem(
id = "wrong-profile",
sessionId = STORED_SESSION_ID,
role = "assistant",
content = JsonPrimitive("Wrong profile history"),
),
)
},
)
}
viewModel.switchProfileContext(
AgentDisplay.profileContextKey("connection-a", observerProfile.name),
STORED_SESSION_ID,
)
awaitCondition { !viewModel.isLoadingHistory.value }
viewModel.setChatVisible(false)
viewModel.updateGatewayClient(null)
gatewayClient.shutdown()
gatewayScope.cancel()
val ownershipMethods = setOf(
"session.resume",
"session.activate",
"session.interrupt",
"prompt.submit",
)
val baseline = ownershipMethods.associateWith { method ->
gatewayHarness.rpcLog.count { it.first == method }
}
val baselineActiveList = gatewayHarness.rpcLog.count { it.first == "session.active_list" }
gatewayHarness.activeSessionListPayload = activeSessionPayload("working")
gatewayScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
gatewayClient = GatewayChatClient(
initialDashboardClient = DashboardApiClient(
baseUrl = gatewayHarness.server.url("/").toString().trimEnd('/'),
okHttpClient = OkHttpClient(),
),
okHttpClient = OkHttpClient(),
callbackDispatcher = { block -> Handler(Looper.getMainLooper()).post(block) },
scope = gatewayScope,
)
viewModel.setChatTurnCheckpointStore(MemoryCheckpointStore())
viewModel.updateGatewayClient(gatewayClient)
viewModel.setChatVisible(true)
viewModel.prewarmGateway()
awaitCondition {
gatewayHarness.rpcLog.count { it.first == "session.active_list" } > baselineActiveList
}
persistedHistory = listOf(
MessageItem(
id = "desktop-answer",
sessionId = STORED_SESSION_ID,
role = "assistant",
content = JsonPrimitive("Desktop completed without Android attachment."),
),
)
awaitCondition {
handler.messages.value.singleOrNull()?.content ==
"Desktop completed without Android attachment."
}
ownershipMethods.forEach { method ->
assertEquals(
"passive foreground sent $method",
baseline.getValue(method),
gatewayHarness.rpcLog.count { it.first == method },
)
}
viewModel.updateGatewayClient(null)
gatewayClient.shutdown()
assertEquals(
"observer teardown interrupted the Desktop turn",
baseline.getValue("session.interrupt"),
gatewayHarness.rpcLog.count { it.first == "session.interrupt" },
)
}
@Test
fun observerReadyAfterChatHidesCannotRestartPassiveWork() {
viewModel.switchProfileContext(PROFILE_CONTEXT, STORED_SESSION_ID)
awaitCondition { !viewModel.isLoadingHistory.value }
viewModel.setChatVisible(false)
viewModel.updateGatewayClient(null)
gatewayClient.shutdown()
gatewayScope.cancel()
val historyReads = AtomicInteger(0)
viewModel.setProfileMessageLoader {
historyReads.incrementAndGet()
Result.success(persistedHistory)
}
val controlMethods = setOf(
"session.resume",
"session.activate",
"session.interrupt",
"prompt.submit",
)
val baseline = controlMethods.associateWith { method ->
gatewayHarness.rpcLog.count { it.first == method }
}
val baselineActiveList = gatewayHarness.rpcLog.count { it.first == "session.active_list" }
gatewayHarness.suppressGatewayReady = true
gatewayScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
gatewayClient = GatewayChatClient(
initialDashboardClient = DashboardApiClient(
baseUrl = gatewayHarness.server.url("/").toString().trimEnd('/'),
okHttpClient = OkHttpClient(),
),
okHttpClient = OkHttpClient(),
callbackDispatcher = { block -> Handler(Looper.getMainLooper()).post(block) },
scope = gatewayScope,
)
viewModel.setChatTurnCheckpointStore(MemoryCheckpointStore())
viewModel.updateGatewayClient(gatewayClient)
viewModel.setChatVisible(true)
val delayedSocket = gatewayHarness.awaitServerSocket()
viewModel.setChatVisible(false)
gatewayHarness.sendGatewayReady(delayedSocket)
shadowOf(Looper.getMainLooper()).idleFor(500, TimeUnit.MILLISECONDS)
Thread.sleep(100)
assertEquals(0, historyReads.get())
assertEquals(
baselineActiveList,
gatewayHarness.rpcLog.count { it.first == "session.active_list" },
)
controlMethods.forEach { method ->
assertEquals(baseline.getValue(method), gatewayHarness.rpcLog.count { it.first == method })
}
}
@Test
fun staleHistoryReadCannotEraseATurnCompletedDuringTheFetch() {
val loadCount = AtomicInteger(0)
@@ -1,6 +1,9 @@
package com.hermesandroid.relay.viewmodel
import com.hermesandroid.relay.assistant.AssistantSessionNotice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
@@ -20,4 +23,35 @@ class VoiceCaptureGuardTest {
fun acceptsSettledShortUtterance() {
assertFalse(shouldDiscardVoiceCapture(durationMs = 420L, pcmBytes = 12_000))
}
@Test
fun noSpeechReturnsToRetryableIdleWithDurableFeedback() {
val state = voiceNoSpeechState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Transcribing,
transcribedText = "stale",
)
)
assertEquals(VoiceState.Idle, state.state)
assertEquals(AssistantSessionNotice.NoSpeech, state.assistantNotice)
assertNull(state.error)
assertNull(state.transcribedText)
assertTrue(state.voiceMode)
}
@Test
fun unrelatedCaptureCancellationDoesNotClaimNoSpeech() {
val state = voiceCaptureCancellationState(
VoiceUiState(
voiceMode = true,
state = VoiceState.Listening,
assistantNotice = AssistantSessionNotice.NoSpeech,
)
)
assertEquals(VoiceState.Idle, state.state)
assertNull(state.assistantNotice)
}
}
+5 -33
View File
@@ -2495,6 +2495,8 @@ boundary.
- Activation heartbeats let the main runtime clean up after assistant-process loss.
Finish and show-failure paths clear pending/watchdog state, while Full Voice
explicitly transfers ownership before the session overlay stops heartbeats.
A recreated session process requests the current activation-fenced voice
snapshot rather than treating its empty local state as authoritative.
- Connection, chat, and voice runtime ownership is application-lifetime in the
main process rather than Activity-owned. The assistant service may initialize
that graph and start a turn while no Activity exists; the app UI later binds
@@ -2508,6 +2510,9 @@ boundary.
- Background and locked-screen invocation is mediated by Android's selected
assistant UI/session rather than an ordinary background Activity launch.
- Locked assistant UI exposes only generic phase and retry status. Transcript,
response, route-specific errors, diagnostics, and screen context remain hidden
until the device is unlocked; no-speech retry copy is deliberately content-free.
- Users can leave Hermes selected for gesture/power-button invocation while
turning continuous KWS off, or remove Hermes through Android's Assistant
settings.
@@ -3977,36 +3982,3 @@ disappearance, client-side profile isolation, and method-not-found; physical
and current-host certification remains tracked in `TODO.md`. An upstream
profile field/filter or explicitly owned aggregate activity route would remove
the remaining ambiguity for multi-profile clients.
---
## ADR 69 — Passive Android observation never attaches another client's Gateway turn
**Status:** Accepted (2026-08-28).
**Context.** `session.resume` and `session.activate` are live-runtime attachment
operations, not read-only subscriptions. Android previously called
`session.resume` while opening or foregrounding Chat and after loading a saved
session's history. When Desktop/TUI already owned a running turn, that passive
prewarm could rebind the runtime transport to Android. A later Android socket,
route, or client teardown could then strand the producer or promote the foreign
turn into an Android `GatewayTurn` whose cancellation sends `session.interrupt`.
The issue was distinct from the earlier stale-view and missing-terminal recovery
paths, which concern exact Android-owned checkpoints.
**Decision.** Ordinary visibility, foreground restoration, Idle-socket recovery,
and saved-session selection establish only the shared Gateway socket. They use
profile-scoped REST history plus process-wide `session.active_list`; while an
unowned row with the selected durable id is live, Android performs bounded
history refreshes and one final read after settlement. These observer paths send
no `session.resume`, `session.activate`, `prompt.submit`, or `session.interrupt`.
Exact Android-owned checkpoints retain `session.activate` with durable-resume
fallback, and explicit send or session-config actions may resume because the user
is intentionally taking control of that destination.
**Consequences.** Opening Android cannot replace, stop, or later cancel a turn
already running in Desktop/TUI. Live token frames remain with the producing
client; Android observes durable progress and final history without inventing a
multi-subscriber Gateway contract. The first explicit Android mutation may pay
the resume latency that passive prewarm previously hid. Cross-client fixtures
and Android lifecycle coverage enforce the no-control-RPC observation boundary.
-1
View File
@@ -41,7 +41,6 @@ the upstream contract identifiers it depends on.
| `active_status_lifecycle` | `session.active_list` reports starting, working, waiting, and idle, then a complete empty process-wide snapshot permits removal of unambiguously owned prior rows |
| `active_status_profile_scope` | A row has no profile metadata and a caller profile hint has no effect; the client must use exact client-held ownership and reject invented attribution |
| `active_status_unsupported` | An older Gateway returns JSON-RPC method-not-found; the client retains Unknown rather than inventing Idle or Working |
| `cross_client_observation` | A second client observes a Desktop-owned working session through active status and history without resume, activate, submit, or interrupt; the producing client receives the terminal event |
Fixture evidence is a bounded metadata-only ring. It records sequence,
connection number, RPC method, event type, scope classification, and outcome.
+7 -2
View File
@@ -481,7 +481,7 @@ Bottom navigation bar with 4 tabs:
- **Bot group projection** — Android merges the bounded `ui_meta["hermes-bots-groups"]` v3 projection across gateways by durable room identity and newest revision. Rooms and recent messages are visibly read-only; Android does not create, rename, disband, join, send, coordinate member turns, or become a second room-log authority. Binary room images are ignored at this metadata boundary.
- **Session drawer** (swipe from left or hamburger icon) — session list with title, timestamp, message count. Create, switch, rename, delete, pin/unpin, and archive/restore. The process-owned conversation binding is the single connection/profile/session identity for Chat; selecting an All Profiles row atomically makes its owner the selected agent and persists that profile/session, while merely browsing All Profiles changes no agent state. Lifecycle or locale-driven Activity recreation cannot replace an explicit binding with stale persisted state, and asynchronous list/history/mutation work is accepted only for the binding's exact namespace. A profile lock hides All Profiles and rejects stale/deep-linked cross-profile opens. The All Profiles browser mode otherwise survives Activity state restoration and refetches its rows after recreation. Pin and archive are durable upstream session fields loaded and patched through the owning connection/profile's Dashboard session API; Android does not keep a second local flag registry. Archived rows are requested explicitly so they remain restorable after recreation. Failed mutations roll back the optimistic row, while refresh and deletion reconcile from server truth. When a persisted title is absent, use upstream's first-user-message `preview`, matching the Hermes Desktop session picker; show "Untitled" only when neither value exists.
- **Authoritative session activity** — one composite registry keyed by connection, normalized profile, and durable session id drives the drawer, filters, grouping, animation, accessibility, and the visible composer. Exact pending approval/clarify/sudo/secret/MCP requests produce **Needs input**; the Gateway's process-wide `session.active_list` supplies **Starting**, **Working**, and **Idle**; exact terminal or `session.info {running:false}` can settle the matching generation. Because active-list rows normally have no profile metadata, Android assigns a row only through exact foreground/detached ownership already held by that client, or explicit profile metadata if a future upstream sends it. A bounded REST directory never proves global uniqueness. Unresolved rows create no status. Resolved rows from a partial snapshot may update their exact owners, but disappearance settles a scope only when the successful process-wide snapshot was completely and unambiguously resolved for it. Restart/checkpoint recovery is **Checking**; a failed or unsupported live refresh is **Unavailable**, never inferred Idle. REST `is_active` remains recency metadata only. `process.list` may add a separate **Background work** indicator and never keeps the parent conversation Working. Old socket generations, bare session ids from another profile, and delayed snapshots cannot revive newer settled state.
- **Concurrent Gateway chats** — switching sessions, profiles, drafts, or Threads detaches the visible Android-owned turn without sending `session.interrupt`; each Android-owned running chat keeps a connection/profile/session-scoped checkpoint and reattaches to its live Gateway session when reopened. Opening, foregrounding, or selecting a saved session without that exact checkpoint is read-only observation: Android warms only the socket, reads profile-scoped history, and polls `session.active_list` without `session.resume`, `session.activate`, `prompt.submit`, or `session.interrupt`. A Desktop/TUI-owned turn therefore remains owned by its producing client; Android refreshes persisted progress and performs one final history read when the runtime settles. Explicit send/config actions may resume the destination session, explicit Stop still interrupts, and SSE fallback stays single-stream and cancels on navigation.
- **Concurrent Gateway chats** — switching sessions, profiles, drafts, or Threads detaches the visible turn without sending `session.interrupt`; each running chat keeps a connection/profile/session-scoped checkpoint and reattaches to its live Gateway session when reopened. Explicit Stop still interrupts. SSE fallback stays single-stream and cancels on navigation.
- **Queued Gateway follow-ups** — every local queued item is immutably scoped to its originating connection, profile, stored session, transport, and run generation; only that run's completion can make it eligible, and switching sessions shows only that session's queue. Restored text queues retain the same scope, while unavailable/deleted destinations and non-restorable attachment queues fail visibly instead of following the current composer. Drained messages add `queued: true` to `prompt.submit`; ordinary sends omit the field. Authoritative submit rejections (`4004`, `4018`, `4028`, `4029`, `4030`, `4090`, `5008`, `5070`, and `5071`) preserve the server message and never fall through to API-server SSE.
- **Durable composer drafts** — each connection/profile/session owns one app-private draft containing text, quote/edit context, and pending attachment bytes. Metadata and content-addressed blobs live under Android's no-backup directory, are capped at 64 drafts and 128 MB of retained blobs outside the active draft, flush when Chat backgrounds, and are removed after a successful send. Session/profile/connection navigation saves the previous owner before restoring the destination; an opened cross-profile session uses its actual owning profile rather than the global picker.
- **Large paste review** — a default-on Chat setting converts any single insertion of at least 5,000 characters into a visible `pasted-text.txt` attachment before the normal message-length limit rejects it. Gateway uses upstream `file.attach`; API-server SSE and proactive Thread paths materialize the same UTF-8 text into the outgoing prompt and remove only the synthetic attachment from that transport, so the behavior never requires Relay or silently drops content.
@@ -1015,7 +1015,10 @@ utilities.
application-lifetime owner, allowing assistant activation to start cold
without constructing or foregrounding `MainActivity`; full Voice later binds
that same runtime. Cancel, error, app/process recreation, and session finish
use the same scoped protocol. The
use the same scoped protocol. Recreated session UI requests an
activation-fenced snapshot from the app runtime, and capture/no-speech exits
retain a generic retry notice instead of collapsing to an unexplained Ready
state. The
wake recorder is released before the established voice recorder opens, and
assistant listening resumes only after the session exits. This mode is
mutually exclusive with the experimental notification-based foreground
@@ -1046,6 +1049,8 @@ utilities.
Realtime Agent sessions do not claim inclusion. The mic control follows the
active voice state, close remains separate, and **Open full voice** explicitly
transfers ownership so assistant-process cleanup cannot cancel the main-app flow.
While keyguard is active, the surface keeps only generic phase and retry copy;
transcript, response, route-specific errors, and screen context remain hidden.
- Stable voice integrates with `ChatViewModel` by **observing** `messages: StateFlow`; transcribed text goes through normal `chatVm.sendMessage(text)` so voice utterances appear as regular user messages in chat history. Experimental Realtime Agent creates a mirrored chat turn and applies broker events directly so tool state, transcript text, assistant deltas, and final responses appear without leaving voice mode.
- `VoiceModeOverlay` — full-screen UI with the MorphingSphere at 60% height in `voiceMode=true`, transcribed + response text, mic button supporting Tap / Hold / Continuous interaction modes.
- The optional `SYSTEM_ALERT_WINDOW` Voice control is user-invoked from an
@@ -100,47 +100,6 @@ class FixtureTestCase(unittest.IsolatedAsyncioTestCase):
self.assertEqual(["user", "assistant"], [row["role"] for row in history["messages"]])
self.assertEqual(2, history["pagination"]["returned"])
async def test_cross_client_observer_never_claims_or_interrupts_producer(self) -> None:
fixture, base_url = await self.start("cross_client_observation")
producer, _ = await self.connect(base_url)
await self.rpc(producer, 1, "session.resume", {"session_id": fixture.scenario.stored_session_id})
await producer.receive_json()
await self.rpc(producer, 2, "prompt.submit", {"text": "producer-only content"})
producer_frames = await self.frames_until(
producer,
lambda frame: frame.get("params", {}).get("type") == "message.delta",
)
observer, _ = await self.connect(base_url)
await self.rpc(observer, 3, "session.active_list")
active = (await observer.receive_json())["result"]["sessions"]
self.assertEqual("working", active[0]["status"])
async with self.session.get(
f"{base_url}/api/sessions/{fixture.scenario.stored_session_id}/messages",
params={"profile": "default", "limit": 500, "offset": 0, "order": "asc"},
) as response:
self.assertEqual(200, response.status)
self.assertIsInstance((await response.json())["messages"], list)
await observer.close()
producer_frames += await self.frames_until(
producer,
lambda frame: frame.get("params", {}).get("type") == "message.complete",
)
self.assertIn(
"message.complete",
[frame.get("params", {}).get("type") for frame in producer_frames],
)
async with self.session.get(f"{base_url}/__fixture__/evidence") as response:
evidence = await response.json()
observer_methods = [
entry.get("method")
for entry in evidence["entries"]
if entry.get("kind") == "rpc" and entry.get("connection") == 2
]
self.assertEqual(["session.active_list"], observer_methods)
self.assertNotIn("session.interrupt", observer_methods)
async def test_rapid_chunks_tools_and_interims_keep_wire_order(self) -> None:
_, base_url = await self.start("rapid_tools_interims")
ws, _ = await self.connect(base_url)
@@ -306,7 +265,6 @@ class ScenarioTestCase(unittest.TestCase):
"active_status_lifecycle",
"active_status_profile_scope",
"active_status_unsupported",
"cross_client_observation",
"ordinary_turn",
"rapid_tools_interims",
"terminal_gap_activate",
@@ -346,10 +304,6 @@ class ScenarioTestCase(unittest.TestCase):
("gateway.settled_session_info",),
load_scenario("terminal_gap_session_info").contract_requirements,
)
self.assertEqual(
("gateway.message_complete", "gateway.session_active_list"),
load_scenario("cross_client_observation").contract_requirements,
)
def test_tls_arguments_must_be_paired(self) -> None:
with contextlib.redirect_stderr(io.StringIO()):
@@ -1,40 +0,0 @@
{
"name": "cross_client_observation",
"live_session_id": "fixture-desktop-live",
"stored_session_id": "fixture-shared-session",
"profile": "default",
"contract_requirements": [
"gateway.message_complete",
"gateway.session_active_list"
],
"initial_history": [],
"turns": [
{
"steps": [
{"op": "event", "type": "message.start"},
{"op": "event", "type": "message.delta", "payload": {"text": "Desktop still owns this turn."}},
{"op": "sleep", "milliseconds": 250},
{
"op": "persist",
"messages": [
{"id": 1, "role": "user", "content": "Desktop prompt.", "timestamp": 1.0},
{"id": 2, "role": "assistant", "content": "Desktop still owns this turn.", "timestamp": 2.0, "finish_reason": "stop"}
]
},
{"op": "set_running", "value": false},
{"op": "event", "type": "message.complete", "payload": {"text": "Desktop still owns this turn.", "status": "complete"}}
]
}
],
"active_list": {
"supported": true,
"snapshots": [
[
{"id": "fixture-desktop-live", "session_key": "fixture-shared-session", "status": "working", "current": false}
],
[
{"id": "fixture-desktop-live", "session_key": "fixture-shared-session", "status": "idle", "current": false}
]
]
}
}