harley/hermes-android
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Native HTTP POST to /api/transcribe — bypasses evalJs size limit, uses WebView cookies for auth

Browse Source
This commit is contained in:
harley
2026-07-07 22:04:30 -04:00
parent e0c41672ab
commit 020441bd38
3 changed files with 243 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/hermes/app/HermesWebAppInterface.kt
+143 -80
View File
@@ -2,22 +2,27 @@ package com.hermes.app
import android.app.NotificationManager
import android.content.Context
import android.content.Intent
import android.media.MediaRecorder
import android.os.Build
import android.os.Bundle
import android.os.Environment
import android.os.VibrationEffect
import android.os.Vibrator
import android.os.VibratorManager
import android.speech.RecognitionListener
import android.speech.RecognizerIntent
import android.speech.SpeechRecognizer
import android.speech.tts.TextToSpeech
import android.webkit.JavascriptInterface
import android.webkit.WebView
import android.widget.Toast
import androidx.core.app.NotificationCompat
import androidx.core.app.NotificationManagerCompat
import java.io.File
import java.io.FileInputStream
import java.io.OutputStream
import java.net.HttpURLConnection
import java.net.URL
import java.util.Locale
import java.util.UUID
import java.util.concurrent.Executors
/**
* JavaScript interface exposed to the Hermes WebUI as `HermesNative`.
@@ -26,17 +31,18 @@ import java.util.Locale
class HermesWebAppInterface(private val context: Context) {
private var tts: TextToSpeech? = null
private var speechRecognizer: SpeechRecognizer? = null
private var webView: WebView? = null
private var mediaRecorder: MediaRecorder? = null
private var audioFile: File? = null
private var currentCallbackId: String? = null
private var isRecording = false
private val executor = Executors.newSingleThreadExecutor()
private val callbackPrefix = "window.__hermesDictationCallback"
fun setWebView(wv: WebView) {
webView = wv
}
/**
* Get the device info for analytics/debugging.
*/
@JavascriptInterface
fun getDeviceInfo(): String {
return """
@@ -50,9 +56,6 @@ class HermesWebAppInterface(private val context: Context) {
""".trimIndent()
}
/**
* Speak text using TTS.
*/
@JavascriptInterface
fun speak(text: String) {
if (tts == null) {
@@ -69,9 +72,6 @@ class HermesWebAppInterface(private val context: Context) {
}
}
/**
* Vibrate the device.
*/
@JavascriptInterface
fun vibrate(durationMs: Int) {
val vibrator = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
@@ -96,17 +96,11 @@ class HermesWebAppInterface(private val context: Context) {
}
}
/**
* Show a toast message.
*/
@JavascriptInterface
fun toast(message: String) {
Toast.makeText(context, message, Toast.LENGTH_SHORT).show()
}
/**
* Set the app badge count.
*/
@JavascriptInterface
fun setBadge(count: Int) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
@@ -130,10 +124,6 @@ class HermesWebAppInterface(private val context: Context) {
}
}
/**
* Show a native notification from the web page.
* Called via HermesNative.notify(title, body)
*/
@JavascriptInterface
fun notify(title: String, body: String) {
try {
@@ -154,85 +144,158 @@ class HermesWebAppInterface(private val context: Context) {
}
/**
* Start native speech recognition and return result via callback.
* Start audio recording. Records until stopDictation() is called.
* Called from JS: HermesNative.startDictation(callbackId)
*/
@JavascriptInterface
fun startDictation(callbackId: String) {
try {
if (speechRecognizer != null) {
speechRecognizer?.destroy()
speechRecognizer = null
}
android.util.Log.d("HermesWV", "startDictation: $callbackId")
currentCallbackId = callbackId
isRecording = true
speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context)
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.getDefault())
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
}
android.os.Handler(android.os.Looper.getMainLooper()).post {
try {
// Ensure any previous recognizer/recorder is fully torn down
try { mediaRecorder?.apply { stop(); release() } } catch (_: Exception) {}
mediaRecorder = null
speechRecognizer?.setRecognitionListener(object : RecognitionListener {
override fun onReadyForSpeech(params: Bundle?) {
evalJs("$callbackPrefix('$callbackId', 'READY', '')")
}
override fun onBeginningOfSpeech() {}
override fun onRmsChanged(rmsdB: Float) {}
override fun onBufferReceived(buffer: ByteArray?) {}
override fun onEndOfSpeech() {}
override fun onError(error: Int) {
val msg = when (error) {
SpeechRecognizer.ERROR_NO_MATCH -> "NO_MATCH"
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "TIMEOUT"
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "BUSY"
else -> "ERROR"
audioFile = File(context.cacheDir, "hermes_audio_${UUID.randomUUID()}.webm")
mediaRecorder = MediaRecorder().apply {
// Use VOICE_COMMUNICATION on Samsung — MIC can route differently
setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
setOutputFormat(MediaRecorder.OutputFormat.WEBM)
setAudioEncoder(MediaRecorder.AudioEncoder.OPUS)
setAudioSamplingRate(16000)
setAudioChannels(1)
setAudioEncodingBitRate(16000)
setOutputFile(audioFile!!.absolutePath)
try { prepare() } catch (e: Exception) {
android.util.Log.e("HermesWV", "prepare failed: ${e.message}")
evalJs("$callbackPrefix('$callbackId', 'ERROR', 'prepare: ${e.message?.replace("'", "\\'") ?: "unknown"}')")
return@post
}
evalJs("$callbackPrefix('$callbackId', '$msg', '')")
speechRecognizer?.destroy()
speechRecognizer = null
try { start() } catch (e: Exception) {
android.util.Log.e("HermesWV", "start failed: ${e.message}")
evalJs("$callbackPrefix('$callbackId', 'ERROR', 'start: ${e.message?.replace("'", "\\'") ?: "unknown"}')")
return@post
}
}
android.util.Log.d("HermesWV", "Recording started: ${audioFile!!.path}")
evalJs("$callbackPrefix('$callbackId', 'RECORDING', '')")
} catch (e: Exception) {
android.util.Log.d("HermesWV", "Record start error: ${e.message}")
isRecording = false
evalJs("$callbackPrefix('$callbackId', 'ERROR', '${e.message?.replace("'", "\\'") ?: "unknown"}')")
}
}
}
/**
* Stop recording and send audio to server for transcription.
* Called from JS: HermesNative.stopDictation()
*/
@JavascriptInterface
fun stopDictation() {
android.util.Log.d("HermesWV", "stopDictation")
val id = currentCallbackId ?: return
isRecording = false
android.os.Handler(android.os.Looper.getMainLooper()).post {
try {
mediaRecorder?.apply {
stop()
release()
}
mediaRecorder = null
android.util.Log.d("HermesWV", "Recording stopped")
// Send audio to server for transcription
val file = audioFile ?: return@post
android.util.Log.d("HermesWV", "Audio file size: ${file.length()} bytes at ${file.absolutePath}")
if (!file.exists() || file.length() == 0L) {
evalJs("$callbackPrefix('$id', 'NO_AUDIO', '')")
file.delete()
return@post
}
override fun onResults(results: Bundle?) {
val texts = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
val text = texts?.firstOrNull() ?: ""
val safeText = text.replace("'", "\\'").replace("\n", " ")
evalJs("$callbackPrefix('$callbackId', 'RESULT', '$safeText')")
speechRecognizer?.destroy()
speechRecognizer = null
evalJs("$callbackPrefix('$id', 'TRANSCRIBING', '')")
executor.execute { transcribeAudio(id, file) }
} catch (e: Exception) {
android.util.Log.d("HermesWV", "Record stop error: ${e.message}")
evalJs("$callbackPrefix('$id', 'ERROR', '${e.message?.replace("'", "\\'") ?: "unknown"}')")
}
}
}
private fun transcribeAudio(callbackId: String, file: File) {
try {
val fileBytes = file.readBytes()
if (fileBytes.isEmpty()) {
evalJs("$callbackPrefix('$callbackId', 'NO_AUDIO', '')")
file.delete()
return
}
// POST audio directly to server from native code — avoids evalJs size limits
val prefs = context.getSharedPreferences(AppConstants.PREFS_NAME, Context.MODE_PRIVATE)
val serverUrl = prefs.getString(AppConstants.PREFS_SERVER_URL, AppConstants.DEFAULT_SERVER_URL)
?: AppConstants.DEFAULT_SERVER_URL
val url = URL("$serverUrl/api/transcribe")
val conn = url.openConnection() as HttpURLConnection
conn.requestMethod = "POST"
conn.doOutput = true
conn.setRequestProperty("Content-Type", "audio/webm")
conn.connectTimeout = 30000
conn.readTimeout = 60000
// Copy auth cookies from WebView session
try {
val cookieMgr = android.webkit.CookieManager.getInstance()
val cookies = cookieMgr.getCookie(serverUrl)
if (!cookies.isNullOrEmpty()) {
conn.setRequestProperty("Cookie", cookies)
}
} catch (_: Exception) {}
override fun onPartialResults(partialResults: Bundle?) {
val texts = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
val text = texts?.firstOrNull() ?: ""
val safeText = text.replace("'", "\\'").replace("\n", " ")
evalJs("$callbackPrefix('$callbackId', 'PARTIAL', '$safeText')")
}
conn.outputStream.use { it.write(fileBytes) }
override fun onEvent(eventType: Int, params: Bundle?) {}
})
val responseCode = conn.responseCode
android.util.Log.d("HermesWV", "Transcribe HTTP $responseCode")
speechRecognizer?.startListening(intent)
if (responseCode == 200) {
val body = conn.inputStream.bufferedReader().readText()
android.util.Log.d("HermesWV", "Transcribe OK: $body")
val text = try {
val json = org.json.JSONObject(body)
json.optString("text", json.optString("transcript", body))
} catch (_: Exception) { body }
val safeText = text.replace("'", "\\'").replace("\n", " ")
evalJs("$callbackPrefix('$callbackId', 'RESULT', '$safeText')")
} else {
val errBody = try { conn.errorStream.bufferedReader().readText() } catch (_: Exception) { "HTTP $responseCode" }
android.util.Log.d("HermesWV", "Transcribe failed: $errBody")
evalJs("$callbackPrefix('$callbackId', 'ERROR', '${errBody.replace("'", "\\'")}')")
}
conn.disconnect()
} catch (e: Exception) {
val safeMsg = (e.message ?: "unknown").replace("'", "\\'")
evalJs("$callbackPrefix('$callbackId', 'ERROR', '$safeMsg')")
android.util.Log.e("HermesWV", "Transcribe exception: ${e.message}")
evalJs("$callbackPrefix('$callbackId', 'ERROR', '${e.message?.replace("'", "\\'") ?: "unknown"}')")
} finally {
file.delete()
audioFile = null
}
}
private fun evalJs(js: String) {
try {
webView?.post { webView?.evaluateJavascript(js, null) }
webView?.evaluateJavascript(js, null)
} catch (_: Exception) {}
}
/**
* Clean up TTS resources.
*/
fun shutdown() {
try {
mediaRecorder?.apply { stop(); release() }
} catch (_: Exception) {}
tts?.stop()
tts?.shutdown()
}
app/src/main/java/com/hermes/app/HermesWebViewManager.kt
+93 -28
View File
@@ -1,5 +1,6 @@
package com.hermes.app
import android.app.Activity
import android.content.Context
import android.content.Intent
import android.graphics.Bitmap
@@ -13,11 +14,18 @@ import android.widget.Toast
*/
class HermesWebViewManager(private val context: Context) {
companion object {
const val FILE_CHOOSER_REQUEST_CODE = 10001
}
var webView: WebView? = null
var onPageTitleChanged: ((String) -> Unit)? = null
var onPageLoading: ((Boolean) -> Unit)? = null
var onConnectionLost: (() -> Unit)? = null
// Stored file chooser callback — Activity intercepts in onActivityResult
var filePathCallback: ValueCallback<Array<Uri>>? = null
fun createWebView(): WebView {
return WebView(context).apply {
layoutParams = android.view.ViewGroup.LayoutParams(
@@ -88,37 +96,62 @@ class HermesWebViewManager(private val context: Context) {
return origQuery(desc);
};
}
// Hook into WebUI mic system via native SpeechRecognizer
if (window.HermesNative && window.HermesNative.startDictation) {
// Hook into WebUI mic system via native AudioRecord
if (window.HermesNative && window.HermesNative.startDictation && window.HermesNative.stopDictation) {
console.log('[Hermes] Native audio record available');
var _dictationCallbackId = 0;
var _dictationCallbacks = {};
window.__hermesDictationCallback = function(id, status, text) {
console.log('[Hermes] Dictation cb:', id, status, text);
var cb = _dictationCallbacks[id];
if (cb) { cb(status, text); delete _dictationCallbacks[id]; }
};
// Intercept the mic toggle function
var origToggle = window._toggleMicCapture;
if (origToggle) {
window._toggleMicCapture = function() {
if (window.__hermesDictating) {
window.__hermesDictating = false;
return;
}
var id = 'd' + (++_dictationCallbackId);
window.__hermesDictating = true;
_dictationCallbacks[id] = function(status, text) {
if (status === 'RESULT') {
var ta = document.getElementById('msg');
if (ta) {
ta.value += text;
ta.dispatchEvent(new Event('input', {bubbles: true}));
// Replace mic button to reroute via native audio record
var btn = document.getElementById('btnMic');
if (btn && !btn.dataset.hermesNative) {
console.log('[Hermes] Replacing mic button');
btn.dataset.hermesNative = '1';
var newBtn = btn.cloneNode(true);
btn.parentNode.replaceChild(newBtn, btn);
var _nativeRecording = false;
var _nativeCbId = null;
newBtn.addEventListener('pointerdown', function(e) {
console.log('[Hermes] Mic button pressed, recording:', _nativeRecording);
if (!_nativeRecording) {
// Start recording
_nativeRecording = true;
var id = 'd' + (++_dictationCallbackId);
_nativeCbId = id;
_dictationCallbacks[id] = function(status, text) {
console.log('[Hermes] Dictation result:', status, text);
if (status === 'RECORDING') {
// Recording started, mic stays active
} else if (status === 'RESULT') {
var ta = document.getElementById('msg');
if (ta) {
ta.value += text;
ta.dispatchEvent(new Event('input', {bubbles: true}));
}
_nativeRecording = false;
} else if (status === 'TRANSCRIBING') {
// Server is processing
} else {
_nativeRecording = false;
}
}
window.__hermesDictating = false;
};
HermesNative.startDictation(id);
};
};
console.log('[Hermes] startDictation id=' + id);
HermesNative.startDictation(id);
} else {
// Stop recording and transcribe
console.log('[Hermes] stopDictation');
HermesNative.stopDictation();
}
});
} else {
console.log('[Hermes] Mic button not found or already native');
}
} else {
console.log('[Hermes] Native audio record NOT available');
}
})()
""".trimIndent(), null
@@ -148,8 +181,6 @@ class HermesWebViewManager(private val context: Context) {
handler: SslErrorHandler?,
error: SslError?
) {
// Accept self-signed cert for LAN Hermes server
// We control both ends — this is safe
handler?.proceed()
}
@@ -160,7 +191,6 @@ class HermesWebViewManager(private val context: Context) {
val url = request?.url ?: return false
val host = url.host ?: return false
val serverHost = Uri.parse(getServerUrl())?.host ?: "10.0.1.49"
if (host != serverHost && !host.endsWith(".local")) {
val intent = Intent(Intent.ACTION_VIEW, url)
context.startActivity(intent)
@@ -177,10 +207,32 @@ class HermesWebViewManager(private val context: Context) {
override fun onPermissionRequest(request: PermissionRequest?) {
val resources = request?.resources ?: emptyArray()
// Grant every permission the WebView asks for
// (notification, microphone, camera — we control both ends)
request?.grant(resources)
}
override fun onConsoleMessage(message: ConsoleMessage?): Boolean {
android.util.Log.d("HermesWV", "${message?.message()} (${message?.sourceId()}:${message?.lineNumber()})")
return true
}
override fun onShowFileChooser(
webView: WebView?,
filePathCallback: ValueCallback<Array<Uri>>?,
fileChooserParams: FileChooserParams?
): Boolean {
val intent = fileChooserParams?.createIntent() ?: return false
this@HermesWebViewManager.filePathCallback = filePathCallback
try {
(context as Activity).startActivityForResult(
Intent.createChooser(intent, "Select file"),
FILE_CHOOSER_REQUEST_CODE
)
} catch (e: Exception) {
this@HermesWebViewManager.filePathCallback = null
return false
}
return true
}
}
setDownloadListener { url, userAgent, contentDisposition, mimeType, contentLength ->
@@ -196,6 +248,19 @@ class HermesWebViewManager(private val context: Context) {
}
}
fun onFilePickerResult(resultCode: Int, data: Intent?) {
val cb = filePathCallback ?: return
filePathCallback = null
if (resultCode == Activity.RESULT_OK && data?.data != null) {
cb.onReceiveValue(arrayOf(data.data!!))
} else if (resultCode == Activity.RESULT_OK && data?.clipData != null) {
val uris = Array(data.clipData!!.itemCount) { i -> data.clipData!!.getItemAt(i).uri }
cb.onReceiveValue(uris)
} else {
cb.onReceiveValue(null)
}
}
fun reload() {
webView?.reload()
}
app/src/main/java/com/hermes/app/MainActivity.kt
+7
View File
@@ -195,6 +195,13 @@ class MainActivity : ComponentActivity() {
super.onDestroy()
}
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
super.onActivityResult(requestCode, resultCode, data)
if (requestCode == HermesWebViewManager.FILE_CHOOSER_REQUEST_CODE) {
webViewManager.onFilePickerResult(resultCode, data)
}
}
override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
if (keyCode == KeyEvent.KEYCODE_BACK && webView?.canGoBack() == true) {
webView?.goBack()