Compare commits

..
12 changed files with 519 additions and 521 deletions
+1 -3
View File
@@ -16,9 +16,7 @@ function classifyCiPaths(paths) {
android: forceAll || under(['app/', 'relay-core/', 'relay-ui/', 'ui-preview/', 'quest/', 'gradle/']) || exact([
'build.gradle.kts', 'settings.gradle.kts', 'gradle.properties', 'gradlew', 'gradlew.bat',
'scripts/check-android-locales.py', 'scripts/android-locale-harness.py',
'scripts/check-android-collection-apis.py', 'scripts/check-android-native-compat.py',
'scripts/check-android-release-notes.py',
'scripts/tests/check_android_native_compat_test.py',
'scripts/check-android-collection-apis.py', 'scripts/check-android-release-notes.py',
'scripts/tests/check_android_release_notes_test.py', '.github/workflows/ci-android.yml',
'.github/workflows/play-preflight-android.yml',
'.github/workflows/approve-release-android.yml',
@@ -16,8 +16,6 @@ assert.deepEqual(classifyCiPaths(['README.md']), none);
assert.deepEqual(classifyCiPaths(['desktop/src/cli.ts']), { ...none, desktop: true });
assert.deepEqual(classifyCiPaths(['relay-core/src/main/kotlin/Wire.kt']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['scripts/check-android-release-notes.py']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['scripts/check-android-native-compat.py']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['scripts/tests/check_android_native_compat_test.py']), { ...none, android: true });
assert.deepEqual(classifyCiPaths(['plugin/relay/server.py']), { ...none, plugin: true });
assert.deepEqual(classifyCiPaths(['plugin/dashboard/src/App.tsx']), { ...none, dashboard: true });
assert.deepEqual(classifyCiPaths(['user-docs/index.md']), { ...none, docs: true });
-17
View File
@@ -33,9 +33,7 @@ on:
- "scripts/check-android-locales.py"
- "scripts/android-locale-harness.py"
- "scripts/check-android-collection-apis.py"
- "scripts/check-android-native-compat.py"
- "scripts/check-android-release-notes.py"
- "scripts/tests/check_android_native_compat_test.py"
- "scripts/tests/check_android_release_notes_test.py"
- ".github/workflows/ci-android.yml"
- ".github/workflows/play-preflight-android.yml"
@@ -82,9 +80,6 @@ jobs:
python3 scripts/check-android-release-notes.py
python3 -m unittest scripts.tests.check_android_release_notes_test
- name: Test Android native compatibility checker
run: python3 -m unittest scripts.tests.check_android_native_compat_test
- name: Run Android lint
run: ./gradlew lint --console=plain
@@ -114,12 +109,6 @@ jobs:
- name: Build debug APK
run: ./gradlew assembleDebug --console=plain
- name: Verify packaged ONNX Runtime compatibility
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/googlePlay/debug/*.apk \
app/build/outputs/apk/sideload/debug/*.apk
- name: Upload debug APK
uses: actions/upload-artifact@v7
if: ${{ github.ref == 'refs/heads/main' }}
@@ -236,9 +225,3 @@ jobs:
python3 scripts/check-android-collection-apis.py \
--apk app/build/outputs/apk/googlePlay/release/*.apk \
--apk app/build/outputs/apk/sideload/release/*.apk
- name: Verify packaged ONNX Runtime compatibility
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/googlePlay/release/*.apk \
app/build/outputs/apk/sideload/release/*.apk
-13
View File
@@ -263,19 +263,6 @@ jobs:
python3 scripts/check-android-collection-apis.py \
--apk app/build/outputs/apk/sideload/candidate/*.apk
- name: Verify stable packaged ONNX Runtime compatibility
if: ${{ needs.validate.outputs.prerelease != 'true' }}
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/googlePlay/release/*.apk \
app/build/outputs/apk/sideload/release/*.apk
- name: Verify candidate packaged ONNX Runtime compatibility
if: ${{ needs.validate.outputs.prerelease == 'true' }}
run: |
python3 scripts/check-android-native-compat.py \
app/build/outputs/apk/sideload/candidate/*.apk
- name: List produced artifacts (debug aid)
run: |
echo "=== APK outputs ==="
+1 -1
View File
@@ -17,7 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
### Fixed
- **Android wake-word detection now loads a compatible native ONNX Runtime.** Packaged sherpa and Java JNI consumers are checked against the shared runtime for every supported ABI before release.
- **Android Continuous voice waits for barge-in microphone teardown before listening again.** Multi-turn hands-free conversations no longer lose the microphone after a response finishes with barge-in enabled. (#464)
- **The visible Android Sphere keeps its smooth procedural motion across startup and chat.** Backgrounded and motion-disabled surfaces remain still without reducing foreground animation to a stepped ambient pulse.
### Removed
@@ -257,7 +257,10 @@ private fun classifyErrorInternal(t: Throwable?, context: String?, ctx: Context?
val msg = t.message.orEmpty().lowercase()
if ("cannot create audiorecord" in msg || "audiorecord failed to initialize" in msg) {
if ("cannot create audiorecord" in msg ||
"audiorecord failed to initialize" in msg ||
"microphone is in use by another voice feature" in msg
) {
return HumanError(
title = ctx?.getString(R.string.error_classify_mic_unavailable) ?: "Microphone unavailable",
body = ctx?.getString(R.string.error_classify_mic_unavailable_body)
@@ -90,6 +90,7 @@ import java.util.Collections
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.concurrent.atomic.AtomicReference
import com.hermesandroid.relay.data.VoicePreferencesRepository
import com.hermesandroid.relay.data.VoiceAudioRoute
@@ -1037,6 +1038,14 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
private var bargeInListener: BargeInListener? = null
private var bargeInListenerJob: Job? = null
private var bargeInVadEngine: VadEngine? = null
/**
* The most recent asynchronous AudioRecord shutdown still releasing the
* process-wide BargeIn microphone lease. Teardown is intentionally
* idempotent, so completion paths may call [stopBargeInListener] after the
* listener reference has already been cleared. Retaining this fence makes
* every subsequent VoiceCapture start join the same ownership handoff.
*/
private val pendingBargeInReaderRelease = AtomicReference<Job?>(null)
private val bargeInTurnEpoch = AtomicLong(0L)
@Volatile private var activeBargeInTurnEpoch: Long = 0L
@@ -1373,6 +1382,11 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
return
}
// A mode change supersedes any capture that is still waiting for the
// previous microphone owner to release. The selected mode below may
// start a fresh Continuous capture with its own generation.
cancelPendingListeningStart()
if (mode != InteractionMode.Continuous) {
continuousLoopArmed = false
continuousListeningPaused = false
@@ -1844,6 +1858,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
}
fun exitVoiceMode() {
cancelPendingListeningStart()
// Idempotence guard — added 2026-04-21 after logcat showed the voice-
// exit chime playing on every Add-connection tap.
//
@@ -1995,6 +2010,10 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
// ---------------------------------------------------------------------
fun startListening() {
startListening(requireContinuousLoop = false)
}
private fun startListening(requireContinuousLoop: Boolean) {
// A direct mic tap starts a normal capture. Only the recorder opened by
// onBargeInDetected may carry response-interruption command context.
responseInterruptedForVoiceCommand = false
@@ -2003,7 +2022,11 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
setError("Recorder not initialized")
return
}
if (pendingListeningStartJob?.isActive == true) return
if (requireContinuousLoop && !canStartContinuousCapture()) return
// A direct/new capture request supersedes a stale handoff waiter. It
// will join the same retained microphone-release fence under a fresh
// epoch below instead of being silently dropped.
cancelPendingListeningStart()
if (rec.isRecording()) return
if (_uiState.value.state == VoiceState.Listening) {
// Listening is reserved for a live AudioRecord. Reconcile a stale
@@ -2037,7 +2060,9 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
try { realtimePcmPlayer?.stop() } catch (_: Exception) { /* ignore */ }
if (microphoneRelease == null || microphoneRelease.isCompleted) {
startVoiceCapture(rec)
if (!requireContinuousLoop || canStartContinuousCapture()) {
startVoiceCapture(rec)
}
return
}
@@ -2045,7 +2070,9 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
val pendingStart = viewModelScope.launch(start = CoroutineStart.LAZY) {
try {
microphoneRelease.join()
if (listeningStartEpoch == startEpoch) {
if (listeningStartEpoch == startEpoch &&
(!requireContinuousLoop || canStartContinuousCapture())
) {
startVoiceCapture(rec)
}
} finally {
@@ -2058,6 +2085,15 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
pendingStart.start()
}
private fun canStartContinuousCapture(): Boolean {
val state = _uiState.value
return state.voiceMode &&
state.interactionMode == InteractionMode.Continuous &&
state.state == VoiceState.Idle &&
continuousLoopArmed &&
!continuousListeningPaused
}
private fun startVoiceCapture(rec: VoiceRecorder) {
try {
rec.startRecording()
@@ -2218,6 +2254,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
* listening turn; until then, idle queue-drain callbacks are ignored.
*/
fun pauseContinuousMode() {
cancelPendingListeningStart()
continuousLoopArmed = false
continuousListeningPaused = _uiState.value.interactionMode == InteractionMode.Continuous
continuousResumeJob?.cancel()
@@ -5760,7 +5797,7 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
continuousLoopArmed &&
_uiState.value.state == VoiceState.Idle
) {
startListening()
startListening(requireContinuousLoop = true)
}
}
@@ -5825,6 +5862,29 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
)
return
}
if (!activeResponseOwnsBargeIn()) {
Log.i(TAG, "Barge-in listener skipped; no active voice response owns the microphone")
return
}
val pendingReaderRelease = pendingBargeInReaderRelease.get()?.takeUnless { it.isCompleted }
if (pendingReaderRelease != null) {
// A late playback/realtime callback may request the next turn's
// listener while the previous AudioRecord is still unwinding.
// Join the same ownership fence as VoiceCapture, then re-check the
// turn epoch so stale generations cannot reopen the microphone.
activeBargeInTurnEpoch = epoch
viewModelScope.launch {
pendingReaderRelease.join()
if (activeBargeInTurnEpoch == epoch &&
bargeInListener == null &&
activeResponseOwnsBargeIn()
) {
startBargeInListenerIfEnabled(epoch = epoch)
}
}
return
}
val vad = try {
vadFactory().also { it.setSensitivity(prefs.sensitivity) }
@@ -5873,6 +5933,12 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
}
}
private fun activeResponseOwnsBargeIn(): Boolean {
val state = _uiState.value
return state.voiceMode &&
(state.state == VoiceState.Thinking || state.state == VoiceState.Speaking)
}
/**
* Tear down the active [BargeInListener], cancel its event subscribers,
* unduck the player (in case a ducking watchdog hadn't yet restored
@@ -5906,7 +5972,13 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
try { realtimePcmPlayer?.unduck() } catch (_: Throwable) { /* ignore */ }
isDucked = false
}
return stoppedReaderJob
if (stoppedReaderJob != null) {
pendingBargeInReaderRelease.set(stoppedReaderJob)
stoppedReaderJob.invokeOnCompletion {
pendingBargeInReaderRelease.compareAndSet(stoppedReaderJob, null)
}
}
return pendingBargeInReaderRelease.get()?.takeUnless { it.isCompleted }
}
private fun markBargeInPlaybackStarted(graceMs: Long) {
@@ -5958,6 +6030,8 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
*/
internal fun onBargeInDetected() {
if (isBargeInStartupGuardActive()) return
val interruptedMode = _uiState.value.interactionMode
val interruptedEngine = voiceEngineMode
val interruptedSpokenReply = _uiState.value.outputAudioActive
if (interruptedSpokenReply) spokenInterruptionLatch.mark()
duckingWatchdog?.cancel(); duckingWatchdog = null
@@ -5991,18 +6065,63 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
responseText = "",
)
}
viewModelScope.launch {
val captureEpoch = ++listeningStartEpoch
val pendingStart = viewModelScope.launch(start = CoroutineStart.LAZY) {
try {
microphoneRelease?.join()
val rec = recorder
if (rec != null && !rec.isRecording()) {
if (!canStartBargeInCapture(captureEpoch, interruptedMode, interruptedEngine)) {
abandonBargeInCaptureIfCurrent(captureEpoch)
return@launch
}
val rec = recorder ?: error("Recorder not initialized")
if (!rec.isRecording()) {
rec.startRecording()
}
scheduleResumeWatchdog()
if (canStartBargeInCapture(captureEpoch, interruptedMode, interruptedEngine)) {
scheduleResumeWatchdog()
} else {
try { rec.cancel() } catch (_: Throwable) { /* ignore */ }
abandonBargeInCaptureIfCurrent(captureEpoch)
}
} catch (t: CancellationException) {
abandonBargeInCaptureIfCurrent(captureEpoch)
throw t
} catch (t: Throwable) {
responseInterruptedForVoiceCommand = false
Log.w(TAG, "barge-in microphone handoff failed: ${t.message}")
surfaceError(t, context = "record")
if (listeningStartEpoch == captureEpoch) {
responseInterruptedForVoiceCommand = false
Log.w(TAG, "barge-in microphone handoff failed: ${t.message}")
surfaceError(t, context = "record")
}
} finally {
if (listeningStartEpoch == captureEpoch) {
pendingListeningStartJob = null
}
}
}
pendingListeningStartJob = pendingStart
pendingStart.start()
}
private fun canStartBargeInCapture(
captureEpoch: Long,
interruptedMode: InteractionMode,
interruptedEngine: VoiceEngineMode,
): Boolean {
val state = _uiState.value
return listeningStartEpoch == captureEpoch &&
state.voiceMode &&
state.state == VoiceState.Listening &&
state.interactionMode == interruptedMode &&
voiceEngineMode == interruptedEngine &&
responseInterruptedForVoiceCommand
}
private fun abandonBargeInCaptureIfCurrent(captureEpoch: Long) {
if (listeningStartEpoch != captureEpoch) return
responseInterruptedForVoiceCommand = false
if (_uiState.value.state == VoiceState.Listening && recorder?.isRecording() != true) {
_uiState.update {
it.copy(state = VoiceState.Idle, amplitude = 0f, outputAudioActive = false)
}
}
}
@@ -6314,9 +6433,22 @@ class VoiceViewModel(application: Application) : AndroidViewModel(application) {
startBargeInListenerIfEnabled()
}
@androidx.annotation.VisibleForTesting
internal fun stopBargeInListenerForTest(): Job? = stopBargeInListener()
@androidx.annotation.VisibleForTesting
internal fun finishAgentAudioOutputForTest() {
finishAgentAudioOutput()
}
@androidx.annotation.VisibleForTesting
internal fun setVoiceEngineModeForTest(mode: VoiceEngineMode) {
voiceEngineMode = mode
}
@androidx.annotation.VisibleForTesting
internal fun beginBargeInTurnForTest() {
_uiState.update { it.copy(state = VoiceState.Thinking) }
_uiState.update { it.copy(voiceMode = true, state = VoiceState.Thinking) }
beginBargeInTurnIfEnabled()
}
@@ -152,4 +152,15 @@ class RelayErrorClassifierTest {
assertEquals("Microphone unavailable", err.title)
assertTrue(err.retryable)
}
@Test
fun microphoneOwnershipConflictMapsToRetryableMicUnavailableHint() {
val err = classifyError(
IllegalStateException("Microphone is in use by another voice feature"),
context = "record",
)
assertEquals("Microphone unavailable", err.title)
assertTrue(err.retryable)
}
}
@@ -10,8 +10,10 @@ import com.hermesandroid.relay.audio.VoiceSfxPlayer
import com.hermesandroid.relay.data.BargeInPreferences
import com.hermesandroid.relay.data.BargeInPreferencesRepository
import com.hermesandroid.relay.data.BargeInSensitivity
import com.hermesandroid.relay.data.VoiceEngineMode
import com.hermesandroid.relay.network.relay.RelayVoiceClient
import com.hermesandroid.relay.viewmodel.ChatViewModel
import com.hermesandroid.relay.viewmodel.InteractionMode
import com.hermesandroid.relay.viewmodel.VoiceState
import com.hermesandroid.relay.viewmodel.VoiceViewModel
import io.mockk.every
@@ -273,6 +275,360 @@ class VoiceViewModelBargeInTest {
assertEquals(VoiceState.Idle, vm.uiState.value.state)
}
@Test
fun `continuous completion waits for queue-drain barge-in release before capture`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returnsMany listOf(readerRelease, null, null)
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Finished."), currentIdx = 0)
vm.setInteractionMode(InteractionMode.Continuous)
vm.startBargeInListenerForTest()
runCurrent()
// The play worker tears down barge-in before the shared completion
// finalizer runs. Repeated teardown calls must retain that first
// asynchronous release instead of attempting VoiceCapture immediately.
vm.stopBargeInListenerForTest()
vm.finishAgentAudioOutputForTest()
vm.startBargeInListenerForTest()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
verify(exactly = 1) { bargeInListener.start(any()) }
readerRelease.complete()
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
verify(exactly = 1) { bargeInListener.start(any()) }
assertEquals(VoiceState.Listening, vm.uiState.value.state)
}
@Test
fun `continuous completion repeats serialized handoff across turns`() = runTest {
val firstRelease = Job()
val secondRelease = Job()
var stopCalls = 0
every { bargeInListener.stop() } answers {
when (stopCalls++) {
0 -> firstRelease
1 -> secondRelease
else -> null
}
}
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("First."), currentIdx = 0)
vm.setInteractionMode(InteractionMode.Continuous)
vm.startBargeInListenerForTest()
runCurrent()
vm.stopBargeInListenerForTest()
vm.finishAgentAudioOutputForTest()
runCurrent()
firstRelease.complete()
runCurrent()
vm.seedSpeakingStateForTest(chunks = listOf("Second."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.stopBargeInListenerForTest()
vm.finishAgentAudioOutputForTest()
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
secondRelease.complete()
runCurrent()
verify(exactly = 2) { recorder.startRecording() }
assertEquals(VoiceState.Listening, vm.uiState.value.state)
}
@Test
fun `barge-in disabled keeps continuous completion immediate`() = runTest {
val vm = buildViewModel(
BargeInPreferences(
enabled = false,
sensitivity = BargeInSensitivity.Off,
),
)
vm.seedSpeakingStateForTest(chunks = listOf("Finished."), currentIdx = 0)
vm.setInteractionMode(InteractionMode.Continuous)
vm.finishAgentAudioOutputForTest()
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
assertEquals(VoiceState.Listening, vm.uiState.value.state)
}
@Test
fun `exit cancels continuous capture waiting for microphone release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returnsMany listOf(readerRelease, null, null)
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Finished."), currentIdx = 0)
vm.setInteractionMode(InteractionMode.Continuous)
vm.startBargeInListenerForTest()
vm.stopBargeInListenerForTest()
vm.finishAgentAudioOutputForTest()
runCurrent()
vm.exitVoiceMode()
vm.startBargeInListenerForTest()
readerRelease.complete()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
verify(exactly = 1) { bargeInListener.start(any()) }
assertTrue(!vm.uiState.value.voiceMode)
}
@Test
fun `pause cancels continuous capture waiting for microphone release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returnsMany listOf(readerRelease, null, null)
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Finished."), currentIdx = 0)
vm.setInteractionMode(InteractionMode.Continuous)
vm.startBargeInListenerForTest()
vm.stopBargeInListenerForTest()
vm.finishAgentAudioOutputForTest()
runCurrent()
vm.pauseContinuousMode()
readerRelease.complete()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
assertEquals(VoiceState.Idle, vm.uiState.value.state)
}
@Test
fun `rapid mode change starts only the newly armed continuous capture`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returnsMany listOf(readerRelease, null, null, null)
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Finished."), currentIdx = 0)
vm.setInteractionMode(InteractionMode.Continuous)
vm.startBargeInListenerForTest()
vm.stopBargeInListenerForTest()
vm.finishAgentAudioOutputForTest()
runCurrent()
vm.setInteractionMode(InteractionMode.TapToTalk)
vm.setInteractionMode(InteractionMode.Continuous)
runCurrent()
readerRelease.complete()
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
assertEquals(VoiceState.Listening, vm.uiState.value.state)
}
@Test
fun `next barge-in generation waits for prior reader release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returnsMany listOf(readerRelease, null)
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("First."), currentIdx = 0)
vm.startBargeInListenerForTest()
runCurrent()
vm.stopBargeInListenerForTest()
vm.startBargeInListenerForTest()
vm.startBargeInListenerForTest()
runCurrent()
verify(exactly = 1) { bargeInListener.start(any()) }
readerRelease.complete()
runCurrent()
verify(exactly = 2) { bargeInListener.start(any()) }
}
@Test
fun `barge-in capture waits for reader release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Speaking."), currentIdx = 0)
vm.startBargeInListenerForTest()
runCurrent()
vm.onBargeInDetected()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
readerRelease.complete()
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
assertEquals(VoiceState.Listening, vm.uiState.value.state)
}
@Test
fun `exit invalidates barge-in capture waiting for reader release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Speaking."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.onBargeInDetected()
runCurrent()
vm.exitVoiceMode()
readerRelease.complete()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
assertTrue(!vm.uiState.value.voiceMode)
assertEquals(VoiceState.Idle, vm.uiState.value.state)
}
@Test
fun `continuous pause invalidates barge-in capture waiting for reader release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
val vm = buildViewModel()
vm.setInteractionMode(InteractionMode.Continuous)
vm.seedSpeakingStateForTest(chunks = listOf("Speaking."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.onBargeInDetected()
runCurrent()
vm.pauseContinuousMode()
readerRelease.complete()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
assertEquals(VoiceState.Idle, vm.uiState.value.state)
}
@Test
fun `interaction mode switch invalidates barge-in capture waiting for reader release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Speaking."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.onBargeInDetected()
runCurrent()
vm.setInteractionMode(InteractionMode.HoldToTalk)
readerRelease.complete()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
assertEquals(InteractionMode.HoldToTalk, vm.uiState.value.interactionMode)
assertEquals(VoiceState.Idle, vm.uiState.value.state)
}
@Test
fun `engine switch invalidates barge-in capture waiting for reader release`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Speaking."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.onBargeInDetected()
runCurrent()
vm.setVoiceEngineModeForTest(VoiceEngineMode.RealtimeAgent)
readerRelease.complete()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
assertEquals(VoiceState.Idle, vm.uiState.value.state)
}
@Test
fun `newer manual capture supersedes barge-in release waiter`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Speaking."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.onBargeInDetected()
runCurrent()
vm.startListening()
runCurrent()
verify(exactly = 0) { recorder.startRecording() }
readerRelease.complete()
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
assertEquals(VoiceState.Listening, vm.uiState.value.state)
}
@Test
fun `reader release completion starts valid barge-in capture only once`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Speaking."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.onBargeInDetected()
runCurrent()
assertTrue(readerRelease.complete())
runCurrent()
assertTrue(!readerRelease.complete())
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
assertEquals(VoiceState.Listening, vm.uiState.value.state)
}
@Test
fun `failed barge-in recorder acquisition does not arm resume watchdog`() = runTest {
val readerRelease = Job()
every { bargeInListener.stop() } returns readerRelease
every { recorder.startRecording() } throws
IllegalStateException("Microphone is in use by another voice feature")
val vm = buildViewModel()
vm.seedSpeakingStateForTest(chunks = listOf("Speaking.", "Tail."), currentIdx = 0)
vm.startBargeInListenerForTest()
vm.onBargeInDetected()
runCurrent()
readerRelease.complete()
runCurrent()
advanceTimeBy(700)
runCurrent()
verify(exactly = 1) { recorder.startRecording() }
assertEquals(VoiceState.Error, vm.uiState.value.state)
}
@Test
fun `transient recorder ownership failures remain explicitly retryable`() = runTest {
var attempts = 0
every { recorder.startRecording() } answers {
attempts++
if (attempts <= 2) {
throw IllegalStateException("Microphone is in use by another voice feature")
}
java.io.File("voice-retry-test.wav")
}
val vm = buildViewModel(
BargeInPreferences(
enabled = false,
sensitivity = BargeInSensitivity.Off,
),
)
vm.seedSpeakingStateForTest(chunks = listOf("Finished."), currentIdx = 0)
vm.setInteractionMode(InteractionMode.Continuous)
vm.finishAgentAudioOutputForTest()
runCurrent()
assertEquals(VoiceState.Error, vm.uiState.value.state)
vm.startListening()
runCurrent()
assertEquals(VoiceState.Error, vm.uiState.value.state)
vm.startListening()
runCurrent()
assertEquals(VoiceState.Listening, vm.uiState.value.state)
verify(exactly = 3) { recorder.startRecording() }
}
// -------------------------------------------------------------------
// Test 2 — resume with resumeAfterInterruption=true + silence
// -------------------------------------------------------------------
+1 -1
View File
@@ -31,7 +31,7 @@ play-publisher = "4.1.1"
media3 = "1.11.0"
androidVad = "2.0.10"
sherpaOnnx = "v1.13.4"
onnxRuntime = "1.27.0"
onnxRuntime = "1.29.0"
spatialsdk = "0.13.2"
play-app-update = "2.1.0"
-306
View File
@@ -1,306 +0,0 @@
#!/usr/bin/env python3
"""Verify that packaged Android JNI consumers match their shared ONNX Runtime.
The APK is the authority for this check. Gradle can resolve multiple AARs that
contain the same native filename, and ``pickFirst`` alone does not prove that
the selected runtime exports the symbol version required by sherpa's JNI.
"""
from __future__ import annotations
import argparse
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
import re
import struct
import sys
import zipfile
ORT_ENTRY_POINT = "OrtGetApiBase"
RUNTIME_LIBRARY = "libonnxruntime.so"
REQUIRED_CONSUMER = "libsherpa-onnx-jni.so"
ORT_JAVA_CONSUMER = "libonnxruntime4j_jni.so"
SUPPORTED_ABIS = {"arm64-v8a", "armeabi-v7a", "x86", "x86_64"}
LIBRARY_PATH = re.compile(
r"^(?:base/)?lib/(?P<abi>[^/]+)/(?P<library>[^/]+\.so)$"
)
@dataclass(frozen=True)
class Section:
section_type: int
offset: int
size: int
link: int
entry_size: int
@dataclass(frozen=True)
class VersionedSymbol:
defined: bool
version: str | None
def _unpack(fmt: str, data: bytes, offset: int) -> tuple[int, ...]:
size = struct.calcsize(fmt)
if offset < 0 or offset + size > len(data):
raise ValueError("ELF structure extends beyond the file")
return struct.unpack_from(fmt, data, offset)
def _cstring(data: bytes, offset: int) -> str:
if offset < 0 or offset >= len(data):
raise ValueError("ELF string offset is outside its string table")
end = data.find(b"\0", offset)
if end < 0:
raise ValueError("ELF string is not NUL-terminated")
return data[offset:end].decode("utf-8", errors="replace")
def _elf_layout(data: bytes) -> tuple[str, bool, list[Section]]:
if len(data) < 16 or data[:4] != b"\x7fELF":
raise ValueError("not an ELF file")
elf_class = data[4]
byte_order = data[5]
if elf_class not in (1, 2) or byte_order not in (1, 2):
raise ValueError("unsupported ELF class or byte order")
endian = "<" if byte_order == 1 else ">"
is_64_bit = elf_class == 2
if is_64_bit:
header = _unpack(endian + "HHIQQQIHHHHHH", data, 16)
section_offset, section_entry_size, section_count = header[5], header[10], header[11]
section_format = endian + "IIQQQQIIQQ"
else:
header = _unpack(endian + "HHIIIIIHHHHHH", data, 16)
section_offset, section_entry_size, section_count = header[5], header[10], header[11]
section_format = endian + "IIIIIIIIII"
minimum_entry_size = struct.calcsize(section_format)
if section_entry_size < minimum_entry_size:
raise ValueError("ELF section-header entry is too small")
sections: list[Section] = []
for index in range(section_count):
fields = _unpack(section_format, data, section_offset + index * section_entry_size)
sections.append(
Section(
section_type=fields[1],
offset=fields[4],
size=fields[5],
link=fields[6],
entry_size=fields[9],
)
)
return endian, is_64_bit, sections
def _section_data(data: bytes, section: Section) -> bytes:
end = section.offset + section.size
if section.offset < 0 or end > len(data):
raise ValueError("ELF section extends beyond the file")
return data[section.offset:end]
def _version_names(
data: bytes,
endian: str,
sections: list[Section],
) -> dict[int, str]:
names: dict[int, str] = {}
for section in sections:
if section.section_type not in (0x6FFFFFFD, 0x6FFFFFFE):
continue
if section.link >= len(sections):
raise ValueError("ELF version section has an invalid string-table link")
strings = _section_data(data, sections[section.link])
cursor = 0
while cursor < section.size:
base = section.offset + cursor
if section.section_type == 0x6FFFFFFD: # SHT_GNU_verdef
fields = _unpack(endian + "HHHHIII", data, base)
version_index, aux_offset, next_offset = fields[2], fields[5], fields[6]
name_offset, _ = _unpack(endian + "II", data, base + aux_offset)
names[version_index] = _cstring(strings, name_offset)
else: # SHT_GNU_verneed
fields = _unpack(endian + "HHIII", data, base)
count, aux_offset, next_offset = fields[1], fields[3], fields[4]
aux_cursor = base + aux_offset
for _ in range(count):
aux = _unpack(endian + "IHHII", data, aux_cursor)
names[aux[2] & 0x7FFF] = _cstring(strings, aux[3])
if aux[4] == 0:
break
aux_cursor += aux[4]
if next_offset == 0:
break
cursor += next_offset
return names
def read_versioned_symbols(data: bytes) -> dict[str, list[VersionedSymbol]]:
endian, is_64_bit, sections = _elf_layout(data)
version_names = _version_names(data, endian, sections)
symbols: dict[str, list[VersionedSymbol]] = defaultdict(list)
for dynsym_index, dynsym in enumerate(sections):
if dynsym.section_type != 11: # SHT_DYNSYM
continue
if dynsym.link >= len(sections):
raise ValueError("ELF dynamic-symbol table has an invalid string-table link")
strings = _section_data(data, sections[dynsym.link])
symbol_format = endian + ("IBBHQQ" if is_64_bit else "IIIBBH")
symbol_size = dynsym.entry_size or struct.calcsize(symbol_format)
symbol_count = dynsym.size // symbol_size
versions: tuple[int, ...] = ()
for section in sections:
if section.section_type == 0x6FFFFFFF and section.link == dynsym_index:
raw_versions = _section_data(data, section)
versions = struct.unpack(endian + f"{len(raw_versions) // 2}H", raw_versions)
break
for index in range(symbol_count):
fields = _unpack(symbol_format, data, dynsym.offset + index * symbol_size)
name_offset = fields[0]
section_index = fields[3] if is_64_bit else fields[5]
name = _cstring(strings, name_offset)
if not name:
continue
version_index = (versions[index] & 0x7FFF) if index < len(versions) else 0
symbols[name].append(
VersionedSymbol(
defined=section_index != 0,
version=version_names.get(version_index),
)
)
return symbols
def _single_symbol_version(
blob: bytes,
*,
defined: bool,
context: str,
) -> str:
matches = [
symbol.version
for symbol in read_versioned_symbols(blob).get(ORT_ENTRY_POINT, [])
if symbol.defined == defined
]
if not matches:
role = "export" if defined else "requirement"
raise ValueError(f"{context} has no {ORT_ENTRY_POINT} {role}")
versions = set(matches)
if None in versions:
raise ValueError(f"{context} uses an unversioned {ORT_ENTRY_POINT} symbol")
if len(versions) != 1:
raise ValueError(f"{context} has ambiguous {ORT_ENTRY_POINT} versions: {sorted(versions)}")
return next(iter(versions)) # type: ignore[return-value]
def check_artifact(path: Path, expected_abis: set[str] | None = None) -> list[str]:
failures: list[str] = []
with zipfile.ZipFile(path) as archive:
libraries: dict[str, dict[str, list[zipfile.ZipInfo]]] = defaultdict(
lambda: defaultdict(list)
)
for info in archive.infolist():
match = LIBRARY_PATH.fullmatch(info.filename)
if match:
libraries[match.group("abi")][match.group("library")].append(info)
if not libraries:
return [f"{path}: no packaged native libraries found"]
expected = SUPPORTED_ABIS if expected_abis is None else expected_abis
actual = set(libraries)
if actual != expected:
failures.append(
f"{path.name}: packaged ABI set is {sorted(actual)}, expected {sorted(expected)}"
)
for abi, by_name in sorted(libraries.items()):
for required in (RUNTIME_LIBRARY, REQUIRED_CONSUMER, ORT_JAVA_CONSUMER):
count = len(by_name.get(required, []))
if count != 1:
failures.append(
f"{path.name} [{abi}]: expected exactly one {required}, found {count}"
)
if failures and (
len(by_name.get(RUNTIME_LIBRARY, [])) != 1
or len(by_name.get(REQUIRED_CONSUMER, [])) != 1
or len(by_name.get(ORT_JAVA_CONSUMER, [])) != 1
):
continue
try:
runtime_version = _single_symbol_version(
archive.read(by_name[RUNTIME_LIBRARY][0]),
defined=True,
context=f"{path.name} [{abi}] {RUNTIME_LIBRARY}",
)
for consumer in (REQUIRED_CONSUMER, ORT_JAVA_CONSUMER):
consumer_version = _single_symbol_version(
archive.read(by_name[consumer][0]),
defined=False,
context=f"{path.name} [{abi}] {consumer}",
)
if runtime_version != consumer_version:
failures.append(
f"{path.name} [{abi}]: {consumer} requires "
f"{ORT_ENTRY_POINT}@{consumer_version}, but {RUNTIME_LIBRARY} exports "
f"{ORT_ENTRY_POINT}@{runtime_version}"
)
except ValueError as error:
failures.append(str(error))
return failures
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"artifact",
nargs="+",
type=Path,
help="APK or AAB artifact to inspect.",
)
parser.add_argument(
"--expected-abi",
action="append",
default=[],
help=(
"Expected packaged ABI; repeat to override the standard four-ABI set "
"for a deliberate -Phermes.devAbi build."
),
)
args = parser.parse_args()
failures: list[str] = []
for artifact in args.artifact:
if not artifact.is_file():
failures.append(f"artifact does not exist: {artifact}")
continue
try:
expected_abis = set(args.expected_abi) or SUPPORTED_ABIS
failures.extend(check_artifact(artifact, expected_abis))
except (OSError, ValueError, zipfile.BadZipFile) as error:
failures.append(f"{artifact}: {error}")
if failures:
print("Android native compatibility check failed:", file=sys.stderr)
for failure in failures:
print(f" {failure}", file=sys.stderr)
return 1
print(
"Android native compatibility check passed "
f"({len(args.artifact)} artifact(s), {ORT_ENTRY_POINT} symbol versions aligned)"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -1,164 +0,0 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import struct
import sys
import tempfile
import unittest
import zipfile
SCRIPT = Path(__file__).resolve().parents[1] / "check-android-native-compat.py"
SPEC = importlib.util.spec_from_file_location("check_android_native_compat", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
native_compat = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = native_compat
SPEC.loader.exec_module(native_compat)
def elf_with_ort_symbol(*, defined: bool, version: str) -> bytes:
"""Build a minimal ELF64 containing a versioned OrtGetApiBase dynsym."""
symbol_name = b"OrtGetApiBase"
version_name = version.encode("ascii")
dependency_name = b"libonnxruntime.so"
strings = b"\0" + symbol_name + b"\0" + version_name + b"\0" + dependency_name + b"\0"
symbol_offset = 1
version_offset = symbol_offset + len(symbol_name) + 1
dependency_offset = version_offset + len(version_name) + 1
dynsym = b"\0" * 24 + struct.pack(
"<IBBHQQ",
symbol_offset,
0x12,
0,
1 if defined else 0,
0,
0,
)
versym = struct.pack("<HH", 0, 2)
if defined:
version_section_type = 0x6FFFFFFD
version_data = struct.pack("<HHHHIII", 1, 0, 2, 1, 0, 20, 0)
version_data += struct.pack("<II", version_offset, 0)
else:
version_section_type = 0x6FFFFFFE
version_data = struct.pack("<HHIII", 1, 1, dependency_offset, 16, 0)
version_data += struct.pack("<IHHII", 0, 0, 2, version_offset, 0)
section_blobs = [b"", strings, dynsym, versym, version_data]
offsets: list[int] = []
image = bytearray(b"\0" * 64)
for blob in section_blobs:
while len(image) % 8:
image.append(0)
offsets.append(len(image))
image.extend(blob)
while len(image) % 8:
image.append(0)
section_header_offset = len(image)
section_headers = [
(0, 0, 0, 0, offsets[0], 0, 0, 0, 0, 0),
(0, 3, 0, 0, offsets[1], len(strings), 0, 0, 1, 0),
(0, 11, 0, 0, offsets[2], len(dynsym), 1, 0, 8, 24),
(0, 0x6FFFFFFF, 0, 0, offsets[3], len(versym), 2, 0, 2, 2),
(0, version_section_type, 0, 0, offsets[4], len(version_data), 1, 0, 4, 0),
]
for header in section_headers:
image.extend(struct.pack("<IIQQQQIIQQ", *header))
ident = b"\x7fELF" + bytes((2, 1, 1, 0)) + b"\0" * 8
header = struct.pack(
"<16sHHIQQQIHHHHHH",
ident,
3,
183,
1,
0,
0,
section_header_offset,
0,
64,
0,
0,
64,
len(section_headers),
0,
)
image[:64] = header
return bytes(image)
def write_artifact(path: Path, *, runtime_version: str, abis: set[str]) -> None:
with zipfile.ZipFile(path, "w") as archive:
for abi in abis:
prefix = f"lib/{abi}/"
archive.writestr(
prefix + native_compat.RUNTIME_LIBRARY,
elf_with_ort_symbol(defined=True, version=runtime_version),
)
for consumer in (
native_compat.REQUIRED_CONSUMER,
native_compat.ORT_JAVA_CONSUMER,
):
archive.writestr(
prefix + consumer,
elf_with_ort_symbol(defined=False, version="VERS_1.27.0"),
)
class AndroidNativeCompatTest(unittest.TestCase):
def test_parses_gnu_definition_and_requirement_versions(self) -> None:
provider = native_compat.read_versioned_symbols(
elf_with_ort_symbol(defined=True, version="VERS_1.27.0")
)
consumer = native_compat.read_versioned_symbols(
elf_with_ort_symbol(defined=False, version="VERS_1.27.0")
)
self.assertEqual(
[native_compat.VersionedSymbol(defined=True, version="VERS_1.27.0")],
provider[native_compat.ORT_ENTRY_POINT],
)
self.assertEqual(
[native_compat.VersionedSymbol(defined=False, version="VERS_1.27.0")],
consumer[native_compat.ORT_ENTRY_POINT],
)
def test_accepts_aligned_runtime_and_both_consumers(self) -> None:
with tempfile.TemporaryDirectory() as directory:
artifact = Path(directory) / "aligned.apk"
write_artifact(
artifact,
runtime_version="VERS_1.27.0",
abis=native_compat.SUPPORTED_ABIS,
)
self.assertEqual([], native_compat.check_artifact(artifact))
def test_rejects_runtime_symbol_version_mismatch(self) -> None:
with tempfile.TemporaryDirectory() as directory:
artifact = Path(directory) / "mismatch.apk"
write_artifact(
artifact,
runtime_version="VERS_1.29.0",
abis=native_compat.SUPPORTED_ABIS,
)
failures = native_compat.check_artifact(artifact)
self.assertTrue(any("requires OrtGetApiBase@VERS_1.27.0" in item for item in failures))
self.assertTrue(any(native_compat.ORT_JAVA_CONSUMER in item for item in failures))
def test_rejects_missing_supported_abi(self) -> None:
with tempfile.TemporaryDirectory() as directory:
artifact = Path(directory) / "missing-abi.apk"
write_artifact(
artifact,
runtime_version="VERS_1.27.0",
abis={"arm64-v8a"},
)
failures = native_compat.check_artifact(artifact)
self.assertTrue(any("packaged ABI set" in item for item in failures))
if __name__ == "__main__":
unittest.main()