feat(#1247): hold-to-record voice notes with server-side transcription (#1250)

Co-authored-by: M57 <41645758+Hy4ri@users.noreply.github.com>
This commit is contained in:
Nikhil NairandM57 authored and GitHub committed 2026-09-25 17:43:52 +03:00
1 parent 0e228dec18
commit ae9003decd
19 files changed
+1619 -89

No files matched your search

@@ -440,12 +440,16 @@ fun ChatScreen(
onAddAttachments = { attachments ->
viewModel.addAttachments(attachments)
},
onVoiceNoteRecorded = { file ->
viewModel.sendVoiceNote(file)
},
onShowMessage = { msg ->
scrollScope.launch {
snackbarHostState.showSnackbar(msg)
}
},
launchExternalActivity = launchExternalActivity,
isTranscribingVoiceNote = state.isTranscribingVoiceNote,
context = context,
)
@@ -901,9 +905,16 @@ fun ChatScreen(
}
},
onMicTap = mediaLaunchers.onMicTap,
isListening = mediaLaunchers.isListening,
onMicHoldStart = mediaLaunchers.onMicHoldStart,
onMicHoldEnd = mediaLaunchers.onMicHoldEnd,
onMicHoldCancel = mediaLaunchers.onMicHoldCancel,
isListening = mediaLaunchers.isListening || state.isTranscribingVoiceNote,
isRecordingVoice = mediaLaunchers.isRecordingVoice,
voiceNoteAmplitude = mediaLaunchers.voiceNoteAmplitude,
onStopGeneration = { viewModel.interruptSession() },
isAgentTyping = state.isAgentTyping,
isMainTurnBusy = state.isMainTurnBusy,
canInterrupt = state.canInterrupt,
isConnected = state.isConnected,
isSessionReady = state.isSessionReady && !timelineState.isHistorical,
sessionPreparationFailed = state.resumeError != null,
@@ -6,6 +6,7 @@ import android.util.Log
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.viewModelScope
import com.m57.hermescontrol.data.local.AuthManager
import com.m57.hermescontrol.data.local.DataScope
import com.m57.hermescontrol.data.local.HermesDatabase
import com.m57.hermescontrol.data.local.SlashUsageStore
import com.m57.hermescontrol.data.model.Attachment
@@ -22,6 +23,7 @@ import com.m57.hermescontrol.data.remote.ApiClient
import com.m57.hermescontrol.data.remote.NetworkResult
import com.m57.hermescontrol.data.remote.OkHttpProvider
import com.m57.hermescontrol.data.remote.safeApiCall
import com.m57.hermescontrol.data.repository.VoiceNoteRepository
import com.m57.hermescontrol.data.session.ActiveSessionHolder
import com.m57.hermescontrol.data.session.ProfileSwitchCoordinator
import com.m57.hermescontrol.data.ws.CommandBlocklist
@@ -66,6 +68,11 @@ private const val TAG = "ChatViewModel"
private const val MESSAGE_PAGE_SIZE = 150
private const val TIMELINE_PAGE_SIZE = 500
private const val HISTORY_WINDOW_SIZE = 120
private const val VOICE_NOTE_OFFLINE_MESSAGE = "Voice note not sent — not connected"
private const val VOICE_NOTE_EMPTY_MESSAGE = "No speech detected in the voice note"
private const val VOICE_NOTE_FAILED_MESSAGE = "Voice note transcription failed"
private const val VOICE_NOTE_UNSENT_MESSAGE =
"Voice note not sent — transcript kept in the input field"
private val REASONING_EFFORT_LEVELS =
setOf("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra")
@@ -135,6 +142,8 @@ data class ChatUiState(
val pendingSends: List<PendingSend> = emptyList(),
val isThinking: Boolean = false,
val thinkingText: String = "",
/** True while a recorded voice note uploads for server-side transcription. */
val isTranscribingVoiceNote: Boolean = false,
val isLoading: Boolean = false,
val isLoadingOlder: Boolean = false,
val hasOlderMessages: Boolean = false,
@@ -255,6 +264,15 @@ data class ChatUiState(
) {
/** Convenience — derived from [connectionStatus]. */
val isConnected: Boolean get() = connectionStatus == ConnectionStatus.CONNECTED
/**
* True only while a generation can actually be interrupted: typing can
* start during session preparation, before any runtime session exists,
* and `session.interrupt` has nothing to stop then. The composer derives
* its Stop affordance from this instead of [isAgentTyping] alone
* (review, PR #1250).
*/
val canInterrupt: Boolean get() = isAgentTyping && isSessionReady
}
data class ChatTimelineState(
@@ -414,6 +432,7 @@ class ChatViewModel(
private val ioDispatcher: kotlinx.coroutines.CoroutineDispatcher = kotlinx.coroutines.Dispatchers.IO,
private val historyDispatcher: kotlinx.coroutines.CoroutineDispatcher = searchDispatcher,
private val sendStore: ChatSendStore = ChatSendStore(application),
private val voiceNoteRepository: VoiceNoteRepository = VoiceNoteRepository(),
) : AndroidViewModel(application) {
constructor(application: Application) : this(application, startCleanup = true)
@@ -2874,6 +2893,113 @@ class ChatViewModel(
return true
}
/** True while a voice-note transcription owns the mic-to-send pipeline. */
private var voiceNoteTranscriptionInFlight = false
/**
* Transcribe a recorded voice note through the dashboard's server-side
* STT relay (`POST /api/audio/transcribe` — the desktop client's voice
* path) and submit the transcript as the next message. The profile's
* configured STT provider answers, not the phone's on-device recognizer.
*
* The recording belongs to the session that was active when it was taken:
* transcription takes long enough for the socket, the profile, or the user
* to move on, so a transcript that can no longer land in that session or
* connection/profile scope is preserved in the composer instead of being
* dropped. Only one
* transcription runs at a time — [voiceNoteTranscriptionInFlight] keeps
* every branch below single-flight (review, PR #1250).
*/
fun sendVoiceNote(file: File) {
if (voiceNoteTranscriptionInFlight) {
file.delete()
return
}
if (!canSubmitMessage()) {
file.delete()
_uiState.update { it.copy(errorMessage = VOICE_NOTE_OFFLINE_MESSAGE) }
return
}
val recordedSessionId = _uiState.value.currentSessionId
val recordedRuntimeSessionId = runtimeSessionId
// The recording also belongs to the connection/profile scope it was
// taken on: both session IDs can still be null/null across a scope
// switch while a replacement session create is pending, so the data
// scope is the deciding identity for that window (review, PR #1250).
val recordedDataScope = AuthManager.currentDataScope()
voiceNoteTranscriptionInFlight = true
viewModelScope.launch {
_uiState.update { it.copy(isTranscribingVoiceNote = true) }
try {
val result =
withContext(ioDispatcher) {
try {
voiceNoteRepository.transcribe(file)
} finally {
file.delete()
}
}
when (result) {
is NetworkResult.Success -> {
val transcript = result.data.trim()
if (transcript.isEmpty()) {
_uiState.update { it.copy(errorMessage = VOICE_NOTE_EMPTY_MESSAGE) }
} else {
deliverVoiceTranscript(
transcript = transcript,
recordedSessionId = recordedSessionId,
recordedRuntimeSessionId = recordedRuntimeSessionId,
recordedDataScope = recordedDataScope,
)
}
}
is NetworkResult.Failure -> {
Log.w(TAG, "Voice note transcription failed: ${result.error.message}")
_uiState.update {
it.copy(errorMessage = "$VOICE_NOTE_FAILED_MESSAGE: ${result.error.message}")
}
}
}
} finally {
// Always release the single-flight flag and the composer's
// transcription state, even if a branch above throws.
voiceNoteTranscriptionInFlight = false
_uiState.update { it.copy(isTranscribingVoiceNote = false) }
}
}
}
/**
* Submits a transcribed voice note, or keeps the transcript in the input
* field when it can no longer land where it was recorded. The recording
* file is already deleted by now, so a rejected send restores the text
* instead of losing it.
*/
private fun deliverVoiceTranscript(
transcript: String,
recordedSessionId: String?,
recordedRuntimeSessionId: String?,
recordedDataScope: DataScope?,
) {
val sessionChanged =
_uiState.value.currentSessionId != recordedSessionId ||
runtimeSessionId != recordedRuntimeSessionId
// A connection/profile switch can leave the session IDs null on both
// sides of the recording (a replacement session create may still be
// pending), so the data scope is the ownership check that catches
// that move (review, PR #1250).
val scopeChanged = AuthManager.currentDataScope() != recordedDataScope
if (sessionChanged || scopeChanged || !sendMessage(transcript)) {
_uiState.update {
it.copy(
composerTextToRestore = transcript,
errorMessage = VOICE_NOTE_UNSENT_MESSAGE,
)
}
}
}
private fun canSubmitMessage(): Boolean =
_uiState.value.pendingReasoningLevel == null &&
wsClient.connectionStatus.value == ConnectionStatus.CONNECTED &&
@@ -0,0 +1,120 @@
package com.m57.hermescontrol.ui.chat
import android.content.Context
import android.media.MediaRecorder
import android.os.Build
import android.os.SystemClock
import android.util.Log
import java.io.File
/**
* Records a hold-to-talk voice note as an MPEG-4/AAC file in the app cache.
*
* One instance drives one in-flight recording; the call site owns UI state and
* decides what to do with the clip. 16 kHz mono AAC keeps the payload small
* (roughly 0.5 MB at [MAX_DURATION_MS]) so it uploads quickly to the
* dashboard's transcription endpoint.
*/
class VoiceNoteRecorder(
private val context: Context,
) {
companion object {
private const val TAG = "VoiceNoteRecorder"
/** Clips shorter than this are treated as accidental and discarded. */
const val MIN_DURATION_MS = 500L
/**
* Hard cap. The call site (ChatMediaLaunchers) owns the timer and
* drives the single stop path — the recorder itself no longer stops
* recording, so MediaRecorder's auto-stop cannot race the
* hold-release stop (review, PR #1250).
*/
const val MAX_DURATION_MS = 120_000
}
private var recorder: MediaRecorder? = null
private var outputFile: File? = null
private var startedAtMs = 0L
val isActive: Boolean
get() = recorder != null
/** Starts recording. Returns false when the microphone is unavailable. */
fun start(): Boolean {
if (recorder != null) return false
val file =
runCatching { File.createTempFile("voice_", ".m4a", context.cacheDir) }
.getOrNull()
?: return false
return try {
val mediaRecorder =
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
MediaRecorder(context)
} else {
@Suppress("DEPRECATION")
MediaRecorder()
}
mediaRecorder.apply {
setAudioSource(MediaRecorder.AudioSource.MIC)
setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
setAudioSamplingRate(16_000)
setAudioEncodingBitRate(32_000)
setAudioChannels(1)
setOutputFile(file.absolutePath)
prepare()
start()
}
recorder = mediaRecorder
outputFile = file
startedAtMs = SystemClock.elapsedRealtime()
Log.i(TAG, "Voice note recording started: ${file.name}")
true
} catch (e: Exception) {
Log.e(TAG, "Failed to start voice note recording", e)
releaseQuietly()
file.delete()
false
}
}
/**
* Stops recording and returns the clip when it is long enough to send.
* Returns null for too-short or failed recordings — the temp file is
* removed in both cases.
*/
fun stop(): File? {
val active = recorder ?: return null
val file = outputFile
val durationMs = SystemClock.elapsedRealtime() - startedAtMs
val stopped = runCatching { active.stop() }.isSuccess
releaseQuietly()
val byteCount = file?.length() ?: -1L
Log.i(TAG, "Voice note stop: durationMs=$durationMs stopped=$stopped bytes=$byteCount")
if (!stopped || file == null || durationMs < MIN_DURATION_MS) {
file?.delete()
return null
}
return file
}
/** Aborts the recording and deletes the clip. */
fun cancel() {
if (recorder != null) {
runCatching { recorder?.stop() }
}
releaseQuietly()
outputFile?.delete()
}
/** Peak amplitude of the current recording window — 0 when idle. */
fun currentAmplitude(): Int = runCatching { recorder?.maxAmplitude ?: 0 }.getOrDefault(0)
private fun releaseQuietly() {
runCatching { recorder?.release() }
recorder = null
outputFile = null
startedAtMs = 0L
}
}
@@ -39,6 +39,7 @@ import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.CompositionLocalProvider
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.State
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
@@ -83,6 +84,7 @@ fun ChatInputBar(
isListening: Boolean,
isAgentTyping: Boolean,
isMainTurnBusy: Boolean = false,
canInterrupt: Boolean = false,
isConnected: Boolean,
commandCatalog: CommandCatalog,
isSessionReady: Boolean,
@@ -108,6 +110,12 @@ fun ChatInputBar(
showModelProvider: Boolean = false,
reasoningWireLevel: String? = null,
pendingReasoningLevel: String? = null,
onMicHoldStart: () -> Unit = {},
onMicHoldEnd: () -> Unit = {},
onMicHoldCancel: () -> Unit = {},
isRecordingVoice: Boolean = false,
voiceNoteAmplitude: State<Float> = remember { mutableStateOf(0f) },
onStopGeneration: () -> Unit = {},
) {
// Allow sending while the agent is mid-turn or awaiting approval: the
// gateway's prompt.submit busy-input policy queues it as the next turn
@@ -269,55 +277,64 @@ fun ChatInputBar(
}
val isInputRtl = inputLayoutDirection == LayoutDirection.Rtl
CompositionLocalProvider(LocalLayoutDirection provides inputLayoutDirection) {
BasicTextField(
value = inputFieldValue,
onValueChange = onInputChange,
modifier =
Modifier
.weight(1f)
.heightIn(min = 42.dp, max = 200.dp)
.padding(vertical = 4.dp)
.testTag("chat_input"),
enabled = isConnected,
textStyle =
MaterialTheme.typography.bodyLarge.copy(
color = palette.text,
textAlign = if (isInputRtl) TextAlign.Right else TextAlign.Left,
textDirection = if (isInputRtl) TextDirection.Rtl else TextDirection.Ltr,
),
singleLine = false,
maxLines = 8,
cursorBrush = SolidColor(palette.text),
decorationBox = { innerTextField ->
CompositionLocalProvider(LocalLayoutDirection provides ambientLayoutDirection) {
Box(
modifier = Modifier.fillMaxWidth(),
contentAlignment =
if (isInputRtl) {
Alignment.CenterEnd
} else {
Alignment.CenterStart
},
) {
CompositionLocalProvider(LocalLayoutDirection provides inputLayoutDirection) {
if (inputFieldValue.text.isEmpty()) {
Text(
text = placeholderText,
style = MaterialTheme.typography.bodyLarge,
textAlign = if (isInputRtl) TextAlign.Right else TextAlign.Left,
color = palette.placeholder,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.fillMaxWidth(),
)
if (isRecordingVoice) {
VoiceNoteRecordingPanel(
amplitude = voiceNoteAmplitude,
modifier = Modifier.weight(1f),
)
} else {
CompositionLocalProvider(LocalLayoutDirection provides inputLayoutDirection) {
BasicTextField(
value = inputFieldValue,
onValueChange = onInputChange,
modifier =
Modifier
.weight(1f)
.heightIn(min = 42.dp, max = 200.dp)
.padding(vertical = 4.dp)
.testTag("chat_input"),
enabled = isConnected,
textStyle =
MaterialTheme.typography.bodyLarge.copy(
color = palette.text,
textAlign = if (isInputRtl) TextAlign.Right else TextAlign.Left,
textDirection = if (isInputRtl) TextDirection.Rtl else TextDirection.Ltr,
),
singleLine = false,
maxLines = 8,
cursorBrush = SolidColor(palette.text),
decorationBox = { innerTextField ->
CompositionLocalProvider(LocalLayoutDirection provides ambientLayoutDirection) {
Box(
modifier = Modifier.fillMaxWidth(),
contentAlignment =
if (isInputRtl) {
Alignment.CenterEnd
} else {
Alignment.CenterStart
},
) {
CompositionLocalProvider(
LocalLayoutDirection provides inputLayoutDirection,
) {
if (inputFieldValue.text.isEmpty()) {
Text(
text = placeholderText,
style = MaterialTheme.typography.bodyLarge,
textAlign = if (isInputRtl) TextAlign.Right else TextAlign.Left,
color = palette.placeholder,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.fillMaxWidth(),
)
}
innerTextField()
}
innerTextField()
}
}
}
},
)
},
)
}
}
}
@@ -333,10 +350,15 @@ fun ChatInputBar(
onSend = onSend,
onQueue = { onBusySend(BusySendMode.QUEUE) },
onStopAndSend = { onBusySend(BusySendMode.INTERRUPT) },
canInterrupt = canInterrupt,
onStopGeneration = onStopGeneration,
onAttachTap = { showAttachmentTray = !showAttachmentTray },
onModelTap = onModelTap,
onReasoningSelected = onReasoningTap,
onMicTap = onMicTap,
onMicHoldStart = onMicHoldStart,
onMicHoldEnd = onMicHoldEnd,
onMicHoldCancel = onMicHoldCancel,
modifier = Modifier.testTag("chat_composer_toolbar"),
canDisableReasoning = canDisableReasoning,
supportsReasoning = supportsReasoning,
@@ -12,6 +12,9 @@ import android.util.Log
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.State
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
@@ -27,6 +30,8 @@ import com.m57.hermescontrol.R
import com.m57.hermescontrol.data.model.Attachment
import com.m57.hermescontrol.ui.chat.ChatInputPolicy
import com.m57.hermescontrol.ui.chat.SpeechInputHelper
import com.m57.hermescontrol.ui.chat.VoiceNoteRecorder
import kotlinx.coroutines.delay
import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
@@ -34,10 +39,15 @@ import java.util.Locale
class ChatMediaLaunchers(
val isListening: Boolean,
val isRecordingVoice: Boolean,
val voiceNoteAmplitude: State<Float>,
val onMicTap: () -> Unit,
val onCameraTap: () -> Unit,
val onImageTap: () -> Unit,
val onFileTap: () -> Unit,
val onMicHoldStart: () -> Unit = {},
val onMicHoldEnd: () -> Unit = {},
val onMicHoldCancel: () -> Unit = {},
)
@Composable
@@ -46,8 +56,10 @@ fun rememberChatMediaLaunchers(
onInputFieldValueChange: (TextFieldValue) -> Unit,
onAddAttachment: (uri: String, name: String, mimeType: String, size: Long) -> Unit,
onAddAttachments: (List<Attachment>) -> Unit,
onVoiceNoteRecorded: (file: File) -> Unit,
onShowMessage: (String) -> Unit,
launchExternalActivity: (() -> Unit) -> Unit,
isTranscribingVoiceNote: Boolean,
context: Context = LocalContext.current,
): ChatMediaLaunchers {
var isListening by remember { mutableStateOf(false) }
@@ -57,6 +69,7 @@ fun rememberChatMediaLaunchers(
val currentOnInputFieldValueChange by rememberUpdatedState(onInputFieldValueChange)
val currentOnAddAttachment by rememberUpdatedState(onAddAttachment)
val currentOnAddAttachments by rememberUpdatedState(onAddAttachments)
val currentOnVoiceNoteRecorded by rememberUpdatedState(onVoiceNoteRecorded)
val currentOnShowMessage by rememberUpdatedState(onShowMessage)
val currentLaunchExternalActivity by rememberUpdatedState(launchExternalActivity)
@@ -64,6 +77,59 @@ fun rememberChatMediaLaunchers(
val sttNotAvailableMsg = stringResource(R.string.stt_not_available)
val sttPermissionDeniedMsg = stringResource(R.string.stt_permission_denied)
val cameraErrorMsg = stringResource(R.string.chat_camera_error)
val voiceRecordFailedMsg = stringResource(R.string.chat_voice_record_failed)
// Hold-to-record voice note recorder: the clip uploads to the dashboard
// for server-side transcription, unlike the flat tap path below, which
// still drives the on-device recognizer.
val voiceNoteRecorder = remember { VoiceNoteRecorder(context) }
var isRecordingVoice by remember { mutableStateOf(false) }
// Live mic level for the recording panel — rises fast, decays slowly so
// the meter reads as voice activity instead of flicker.
val voiceNoteAmplitude = remember { mutableStateOf(0f) }
LaunchedEffect(Unit) {
while (true) {
delay(90)
val previous = voiceNoteAmplitude.value
val next =
if (voiceNoteRecorder.isActive) {
maxOf(voiceNoteRecorder.currentAmplitude() / 32767f, previous * 0.8f)
} else {
0f
}
if (next != previous) {
voiceNoteAmplitude.value = next
}
}
}
fun finishVoiceRecording() {
val recordedFile = voiceNoteRecorder.stop()
isRecordingVoice = false
if (recordedFile != null) {
currentOnVoiceNoteRecorded(recordedFile)
}
}
// App-owned max-duration cap: the timer drives the same
// finishVoiceRecording() path as the hold release, so exactly one code
// path ever calls stop() — MediaRecorder's own auto-stop used to race
// the release and discard a valid max-length clip (review, PR #1250).
LaunchedEffect(isRecordingVoice) {
if (isRecordingVoice) {
delay(VoiceNoteRecorder.MAX_DURATION_MS.toLong())
if (voiceNoteRecorder.isActive) {
finishVoiceRecording()
}
}
}
DisposableEffect(voiceNoteRecorder) {
onDispose {
voiceNoteRecorder.cancel()
}
}
// Speech-to-text recognition launcher (issue #194)
val speechLauncher =
@@ -117,6 +183,18 @@ fun rememberChatMediaLaunchers(
}
}
// Hold-to-record permission: unlike the launcher above, granting does NOT
// auto-start dictation — the user records by holding the mic again.
val voiceNotePermissionLauncher =
rememberLauncherForActivityResult(
ActivityResultContracts.RequestPermission(),
) { granted ->
ExternalActivityLifecycleGuard.externalActivityReturned()
if (!granted) {
currentOnShowMessage(sttPermissionDeniedMsg)
}
}
// Multi-file picker for attachments (issue #195)
val filePickerLauncher =
rememberLauncherForActivityResult(
@@ -183,7 +261,14 @@ fun rememberChatMediaLaunchers(
}
val onMicTap: () -> Unit = {
if (isListening) {
if (isTranscribingVoiceNote) {
// Single-flight: a server transcription owns the voice pipeline
// until it lands (review, PR #1250).
} else if (isRecordingVoice) {
// A tap while recording discards the in-flight voice note.
voiceNoteRecorder.cancel()
isRecordingVoice = false
} else if (isListening) {
isListening = false
} else if (
ContextCompat.checkSelfPermission(
@@ -212,6 +297,45 @@ fun rememberChatMediaLaunchers(
}
}
val onMicHoldStart: () -> Unit = {
if (isTranscribingVoiceNote) {
// Single-flight: no new recording while a transcription runs.
} else if (voiceNoteRecorder.isActive || isListening) {
// A voice note or a dictation session is already running.
} else if (
ContextCompat.checkSelfPermission(
context,
Manifest.permission.RECORD_AUDIO,
) == PackageManager.PERMISSION_GRANTED
) {
if (voiceNoteRecorder.start()) {
isRecordingVoice = true
} else {
currentOnShowMessage(voiceRecordFailedMsg)
}
} else {
// The denial message comes from voiceNotePermissionLauncher's
// result callback only — showing it here fired it before the
// dialog even opened (review, PR #1250).
currentLaunchExternalActivity {
voiceNotePermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
}
}
}
val onMicHoldEnd: () -> Unit = {
if (voiceNoteRecorder.isActive) {
finishVoiceRecording()
}
}
val onMicHoldCancel: () -> Unit = {
if (voiceNoteRecorder.isActive) {
voiceNoteRecorder.cancel()
}
isRecordingVoice = false
}
val onCameraTap: () -> Unit = {
try {
val timeStamp =
@@ -245,13 +369,18 @@ fun rememberChatMediaLaunchers(
}
}
return remember(isListening) {
return remember(isListening, isRecordingVoice, isTranscribingVoiceNote) {
ChatMediaLaunchers(
isListening = isListening,
isListening = isListening || isRecordingVoice,
isRecordingVoice = isRecordingVoice,
voiceNoteAmplitude = voiceNoteAmplitude,
onMicTap = onMicTap,
onCameraTap = onCameraTap,
onImageTap = onImageTap,
onFileTap = onFileTap,
onMicHoldStart = onMicHoldStart,
onMicHoldEnd = onMicHoldEnd,
onMicHoldCancel = onMicHoldCancel,
)
}
}
@@ -10,6 +10,8 @@ import androidx.compose.foundation.ExperimentalFoundationApi
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.combinedClickable
import androidx.compose.foundation.gestures.awaitEachGesture
import androidx.compose.foundation.gestures.awaitFirstDown
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
@@ -51,12 +53,16 @@ import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.draw.clipToBounds
import androidx.compose.ui.draw.scale
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.input.pointer.PointerInputScope
import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.platform.testTag
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.semantics.CustomAccessibilityAction
@@ -67,6 +73,7 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.m57.hermescontrol.R
import kotlinx.coroutines.withTimeoutOrNull
/**
* Bottom controls row for the chat composer, rendered inside the composer card.
@@ -116,12 +123,50 @@ fun ComposerToolbar(
reasoningWireLevel: String? = null,
pendingReasoningLevel: String? = null,
isSessionReady: Boolean = true,
canInterrupt: Boolean = false,
onMicHoldStart: () -> Unit = {},
onMicHoldEnd: () -> Unit = {},
onMicHoldCancel: () -> Unit = {},
onStopGeneration: () -> Unit = {},
) {
var showReasoningMenu by remember { mutableStateOf(false) }
val palette = composerPalette()
val reasoningDisabledForModel = supportsReasoning == false
val canDisable = canDisableReasoning
// Telegram-style voice notes: hold to record, release to send, slide away
// to cancel. Both mic controls run the same single pointer loop
// (micHoldHandler), which owns the whole press until finger-up — so the
// release that submits the note can never be lost between recompositions.
val currentIsConnected = rememberUpdatedState(isConnected)
val currentShowSend = rememberUpdatedState(showSend)
val currentCanInterrupt = rememberUpdatedState(canInterrupt)
val currentOnMicHoldStart = rememberUpdatedState(onMicHoldStart)
val currentOnMicHoldEnd = rememberUpdatedState(onMicHoldEnd)
val currentOnMicHoldCancel = rememberUpdatedState(onMicHoldCancel)
val flatMicGesture =
Modifier.pointerInput(Unit) {
micHoldHandler(
isEnabled = {
currentIsConnected.value && currentShowSend.value && !currentCanInterrupt.value
},
onHoldStart = { currentOnMicHoldStart.value() },
onHoldEnd = { currentOnMicHoldEnd.value() },
onHoldCancel = { currentOnMicHoldCancel.value() },
)
}
val actionMicGesture =
Modifier.pointerInput(Unit) {
micHoldHandler(
isEnabled = {
currentIsConnected.value && !currentShowSend.value && !currentCanInterrupt.value
},
onHoldStart = { currentOnMicHoldStart.value() },
onHoldEnd = { currentOnMicHoldEnd.value() },
onHoldCancel = { currentOnMicHoldCancel.value() },
)
}
Row(
modifier =
modifier
@@ -417,25 +462,47 @@ fun ComposerToolbar(
}
}
// Flat mic / stop button — only while the action button is in send mode
// Flat slot — mic while idle; while a generation can be interrupted it
// keeps queue-send available (the action button carries Stop). During
// session preparation the mic stays here and the action button is a
// disabled send: Stop must not appear when there is nothing to
// interrupt (review, PR #1250).
AnimatedVisibility(
visible = showSend,
enter = fadeIn() + scaleIn(initialScale = 0.8f),
exit = fadeOut() + scaleOut(targetScale = 0.8f),
) {
FilledIconButton(
onClick = onMicTap,
enabled = isConnected,
colors = if (isListening) listeningIconButtonColors() else flatIconButtonColors(palette),
modifier =
Modifier
.size(ControlSize)
.testTag(if (isListening) "mic_stop_button" else "mic_button"),
) {
Icon(
imageVector = if (isListening) Icons.Default.Stop else Icons.Outlined.Mic,
contentDescription = if (isListening) "Stop listening" else "Mic",
)
if (canInterrupt) {
FilledIconButton(
onClick = onSend,
enabled = canSend,
colors = flatIconButtonColors(palette),
modifier =
Modifier
.size(ControlSize)
.testTag("send_button"),
) {
Icon(
imageVector = Icons.AutoMirrored.Filled.Send,
contentDescription = stringResource(R.string.chat_send_desc),
)
}
} else {
FilledIconButton(
onClick = onMicTap,
enabled = isConnected,
colors = if (isListening) listeningIconButtonColors() else flatIconButtonColors(palette),
modifier =
Modifier
.size(ControlSize)
.testTag(if (isListening) "mic_stop_button" else "mic_button")
.then(flatMicGesture),
) {
Icon(
imageVector = if (isListening) Icons.Default.Stop else Icons.Outlined.Mic,
contentDescription = if (isListening) "Stop listening" else "Mic",
)
}
}
}
@@ -484,21 +551,29 @@ fun ComposerToolbar(
}
}.combinedClickable(
enabled = actionEnabled,
onClick = if (showSend) onSend else onMicTap,
onClick = {
when {
canInterrupt -> onStopGeneration()
showSend -> onSend()
else -> onMicTap()
}
},
onLongClick = if (showSend && showQueue) onStopAndSend else null,
onLongClickLabel = if (showSend && showQueue) stopAndSendLabel else null,
).testTag(
when {
canInterrupt -> "stop_button"
showSend -> "send_button"
isListening -> "mic_stop_button"
else -> "mic_button"
},
),
).then(if (showSend || canInterrupt) Modifier else actionMicGesture),
contentAlignment = Alignment.Center,
) {
Crossfade(
targetState =
when {
canInterrupt -> ActionGlyph.STOP
showSend -> ActionGlyph.SEND
isListening -> ActionGlyph.STOP
else -> ActionGlyph.VOICE
@@ -517,8 +592,18 @@ fun ComposerToolbar(
ActionGlyph.STOP -> {
Icon(
imageVector = Icons.Default.Stop,
contentDescription = "Stop listening",
tint = if (isListening) MaterialTheme.colorScheme.onErrorContainer else palette.onAction,
contentDescription =
if (canInterrupt) {
stringResource(R.string.chat_voice_stop_generating)
} else {
"Stop listening"
},
tint =
if (isListening && !canInterrupt) {
MaterialTheme.colorScheme.onErrorContainer
} else {
palette.onAction
},
)
}
@@ -632,3 +717,91 @@ fun buildReasoningLabel(
}
return reqLabel
}
/**
* Telegram-style mic gesture: press and hold past [HOLD_TO_RECORD_THRESHOLD_MS]
* records a voice note, release sends it, and sliding away (left or up) from
* the button cancels instead of sending. Short taps use the control's normal
* click handler for dictation, so touch and accessibility actions share one
* callback path.
*
* The whole down-to-up sequence lives in this one pointer loop, so the release
* that submits can never be lost to a recomposition between press and
* finger-up — an earlier interaction-source based approach could lose it and
* leave the recorder running until some later tap "sent" the stale clip.
*/
private suspend fun PointerInputScope.micHoldHandler(
isEnabled: () -> Boolean,
onHoldStart: () -> Unit,
onHoldEnd: () -> Unit,
onHoldCancel: () -> Unit,
) {
val cancelSlop = MIC_SLIDE_CANCEL_DP.dp.toPx()
awaitEachGesture {
val down = awaitFirstDown(requireUnconsumed = false)
if (!isEnabled()) {
return@awaitEachGesture
}
fun slidAway(position: Offset): Boolean {
val dx = position.x - down.position.x
val dy = position.y - down.position.y
return dx < -cancelSlop || dy < -cancelSlop
}
// Phase 1 — a finger-up before the threshold is a plain tap.
var earlyUp = false
var gestureAborted = false
withTimeoutOrNull(HOLD_TO_RECORD_THRESHOLD_MS) {
while (true) {
val event = awaitPointerEvent()
val change =
event.changes.firstOrNull { it.id == down.id }
?: run {
gestureAborted = true
return@withTimeoutOrNull
}
if (!change.pressed) {
earlyUp = true
return@withTimeoutOrNull
}
if (change.isConsumed || slidAway(change.position)) {
gestureAborted = true
return@withTimeoutOrNull
}
}
}
if (gestureAborted) {
return@awaitEachGesture
}
if (earlyUp) {
return@awaitEachGesture
}
// Phase 2 — the threshold passed with the finger down: record until
// the finger lifts (send) or slides away (cancel). No timeout here.
onHoldStart()
var send = false
try {
while (true) {
val event = awaitPointerEvent()
val change = event.changes.firstOrNull { it.id == down.id } ?: break
if (!change.pressed) {
send = true
break
}
if (change.isConsumed || slidAway(change.position)) {
break
}
}
} finally {
if (send) onHoldEnd() else onHoldCancel()
}
}
}
/** A press shorter than this is a tap; longer arms voice-note recording. */
private const val HOLD_TO_RECORD_THRESHOLD_MS = 400L
/** Sliding this far from the mic button (left or up) cancels the recording. */
private const val MIC_SLIDE_CANCEL_DP = 64
@@ -0,0 +1,128 @@
package com.m57.hermescontrol.ui.chat.components
import android.os.SystemClock
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.filled.ArrowBack
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.State
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateListOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.runtime.snapshotFlow
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.testTag
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.unit.dp
import com.m57.hermescontrol.R
import kotlinx.coroutines.delay
/**
* Telegram-style recording strip shown in place of the input field while a
* voice note is being recorded: a slide-to-cancel hint, an elapsed timer, and
* a live mic level meter so the press and the captured audio are both visible
* even when the mic button itself is under the thumb.
*/
@Composable
internal fun VoiceNoteRecordingPanel(
amplitude: State<Float>,
modifier: Modifier = Modifier,
) {
val palette = composerPalette()
var elapsedSeconds by remember { mutableStateOf(0L) }
LaunchedEffect(Unit) {
val startedAt = SystemClock.elapsedRealtime()
while (true) {
elapsedSeconds = (SystemClock.elapsedRealtime() - startedAt) / 1000
delay(200)
}
}
val levels = remember { mutableStateListOf<Float>() }
LaunchedEffect(Unit) {
snapshotFlow { amplitude.value }.collect { level ->
levels.add(level)
while (levels.size > VOICE_NOTE_BARS) {
levels.removeAt(0)
}
}
}
Row(
modifier =
modifier
.heightIn(min = 42.dp)
.testTag("voice_note_recording_panel"),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
Icon(
imageVector = Icons.AutoMirrored.Filled.ArrowBack,
// Decorative: the adjacent "Slide to cancel" text carries the
// instruction, so the arrow must not repeat it (review, PR #1250).
contentDescription = null,
tint = palette.placeholder,
modifier = Modifier.size(16.dp),
)
Text(
text = stringResource(R.string.chat_voice_slide_to_cancel),
style = MaterialTheme.typography.labelMedium,
color = palette.placeholder,
maxLines = 1,
)
Spacer(modifier = Modifier.weight(1f))
Box(
modifier =
Modifier
.size(8.dp)
.background(MaterialTheme.colorScheme.error, CircleShape),
)
Text(
text = formatElapsed(elapsedSeconds),
style = MaterialTheme.typography.labelLarge,
color = palette.text,
)
Row(
modifier = Modifier.height(24.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(2.dp),
) {
val filler = VOICE_NOTE_BARS - levels.size
repeat(VOICE_NOTE_BARS) { index ->
val level = if (index < filler) 0f else levels[index - filler]
Box(
modifier =
Modifier
.width(3.dp)
.height((4f + 18f * level).dp)
.background(palette.placeholder, RoundedCornerShape(2.dp)),
)
}
}
}
}
/** Number of level-meter bars in the recording panel. */
private const val VOICE_NOTE_BARS = 14
/** Formats elapsed recording time as m:ss. */
private fun formatElapsed(seconds: Long): String {
val minutes = seconds / 60
val secs = seconds % 60
return "$minutes:${secs.toString().padStart(2, '0')}"
}