diff --git a/app/src/androidTest/java/com/m57/hermescontrol/data/ws/WsTransportDeviceTest.kt b/app/src/androidTest/java/com/m57/hermescontrol/data/ws/WsTransportDeviceTest.kt new file mode 100644 index 00000000..681267a0 --- /dev/null +++ b/app/src/androidTest/java/com/m57/hermescontrol/data/ws/WsTransportDeviceTest.kt @@ -0,0 +1,296 @@ +package com.m57.hermescontrol.data.ws + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonPrimitive +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okio.ByteString +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +@RunWith(AndroidJUnit4::class) +class WsTransportDeviceTest { + private class FakeWebSocket( + private val request: Request, + var accept: Boolean = true, + var queueSizeValue: Long = 0L, + ) : WebSocket { + val sentFrames = ConcurrentLinkedQueue() + val attempts = AtomicInteger(0) + val closedCode = AtomicInteger(-1) + val closedReason = + java.util.concurrent.atomic + .AtomicReference(null) + val canceled = AtomicBoolean(false) + + override fun request(): Request = request + + override fun queueSize(): Long = queueSizeValue + + override fun send(text: String): Boolean { + attempts.incrementAndGet() + if (!accept) return false + sentFrames.add(text) + return true + } + + override fun send(bytes: ByteString): Boolean { + attempts.incrementAndGet() + if (!accept) return false + sentFrames.add(bytes.utf8()) + return true + } + + override fun close( + code: Int, + reason: String?, + ): Boolean { + closedCode.set(code) + closedReason.set(reason) + return true + } + + override fun cancel() { + canceled.set(true) + } + } + + private class TestFixture { + val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + val lock = Any() + val clock = AtomicLong(100L) + val dummyRequest: Request = Request.Builder().url("http://localhost/ws").build() + val openResponse: Response = + Response + .Builder() + .request(dummyRequest) + .protocol(Protocol.HTTP_1_1) + .code(101) + .message("Switching Protocols") + .build() + + val json = + Json { + ignoreUnknownKeys = true + encodeDefaults = false + } + + val recordedEvents = ConcurrentLinkedQueue() + lateinit var transport: WsTransport + lateinit var rpc: RpcChannel + lateinit var health: ConnectionHealth + lateinit var replay: ReplayTracker + + init { + health = + ConnectionHealth( + scope = scope, + isConnected = { transport.isConnected }, + pingRequest = { _ -> }, + cancelSocket = { transport.cancelSocket() }, + pendingReply = { transport.pendingReply }, + nowMs = { clock.get() }, + ) + + replay = + ReplayTracker( + scope = scope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { recordedEvents.add(it) }, + ) + + rpc = + RpcChannel( + scope = scope, + json = json, + outboundLock = lock, + sendRequest = { method, params, onSent -> + transport.send(rpc.createRequest(method, params), onSent) + }, + sendFrame = { transport.sendDirectFrame(it) }, + removeQueuedMessage = { transport.removeQueuedMessage(it) }, + onIdleCheck = { transport.disconnectIfIdleInBackground() }, + onResult = { transport.onRpcResult(it) }, + onError = { transport.onRpcError(it) }, + ) + + transport = + WsTransport( + scope = scope, + outboundLock = lock, + reconnectPolicy = ReconnectPolicy(), + connectionHealth = health, + replayTracker = replay, + rpcChannel = rpc, + onRawMessage = { raw -> + rpc.handleIncomingFrame(raw, replay) { recordedEvents.add(it) } + }, + ) + } + + fun close() { + try { + transport.disconnect(clearPendingMessages = true) + } finally { + scope.cancel() + } + } + } + + @Test + fun transportRpcRoundTripWithStaleListenerIsolation() = + kotlinx.coroutines.runBlocking { + val fixture = TestFixture() + try { + val fakeWs = FakeWebSocket(fixture.dummyRequest, accept = true, queueSizeValue = 0L) + fixture.transport.connectionGenerationForTest.set(1) + fixture.transport.intentionalCloseForTest.set(false) + + val listenerGen1 = fixture.transport.createListenerForTest(1) + listenerGen1.onOpen(fakeWs, fixture.openResponse) + + assertTrue(fixture.transport.isConnected) + assertEquals(fakeWs, fixture.transport.webSocketForTest) + + val deferred = fixture.rpc.request("gateway.ping", emptyMap()) + val sentFrame = requireNotNull(fakeWs.sentFrames.poll()) + + val parsedSent = fixture.json.decodeFromString(sentFrame) + val reqId = parsedSent.id + + fixture.clock.set(200L) + val responseJson = """{"jsonrpc":"2.0","id":"$reqId","result":42.0}""" + listenerGen1.onMessage(fakeWs, responseJson) + + val result = withTimeout(2_000L) { deferred.await() } + assertEquals(JsonPrimitive(42.0), result) + + val event = requireNotNull(fixture.recordedEvents.poll()) + assertTrue(event is WsEvent.RpcResult) + assertEquals(reqId, (event as WsEvent.RpcResult).id) + assertEquals(200L, fixture.health.lastPongTimestamp) + + val staleListener = fixture.transport.createListenerForTest(0) + val staleWs = FakeWebSocket(fixture.dummyRequest, accept = true) + staleListener.onMessage(staleWs, """{"jsonrpc":"2.0","id":999,"result":99.0}""") + listenerGen1.onMessage(staleWs, """{"jsonrpc":"2.0","id":998,"result":88.0}""") + + assertTrue(fixture.transport.isConnected) + assertEquals(fakeWs, fixture.transport.webSocketForTest) + assertEquals(200L, fixture.health.lastPongTimestamp) + assertTrue(fixture.recordedEvents.isEmpty()) + } finally { + fixture.close() + } + } + + @Test + fun rejectingWsQueuesAndFlushesOnNextAcceptSocket() = + kotlinx.coroutines.runBlocking { + val fixture = TestFixture() + try { + val rejectingWs = FakeWebSocket(fixture.dummyRequest, accept = false, queueSizeValue = 1L) + fixture.transport.connectionGenerationForTest.set(1) + fixture.transport.intentionalCloseForTest.set(false) + + val listener1 = fixture.transport.createListenerForTest(1) + listener1.onOpen(rejectingWs, fixture.openResponse) + assertTrue(fixture.transport.isConnected) + + var recordedId: String? = null + var callbackRanUnderLock = false + val req1 = fixture.rpc.createRequest("test.one", emptyMap()) + val returnedId = + fixture.transport.send( + request = req1, + onSent = { id -> + callbackRanUnderLock = Thread.holdsLock(fixture.lock) + recordedId = id + }, + ) + + assertEquals(returnedId, req1.id.toString()) + assertEquals(returnedId, recordedId) + assertTrue(callbackRanUnderLock) + + val req2 = fixture.rpc.createRequest("test.two", emptyMap()) + fixture.transport.messageQueueForTest.add(fixture.rpc.encodeRequest(req2)) + + assertFalse(fixture.transport.isConnected) + assertEquals(ConnectionStatus.RECONNECTING, fixture.transport.connectionStatus.value) + assertNotNull(fixture.transport.outboundDrainJobForTest) + assertEquals(2, fixture.transport.messageQueueForTest.size) + + val acceptWs = FakeWebSocket(fixture.dummyRequest, accept = true, queueSizeValue = 0L) + val listenerReplacement = fixture.transport.createListenerForTest(1) + listenerReplacement.onOpen(acceptWs, fixture.openResponse) + + assertTrue(fixture.transport.isConnected) + assertEquals(ConnectionStatus.CONNECTED, fixture.transport.connectionStatus.value) + assertNull(fixture.transport.outboundDrainJobForTest) + + assertTrue(fixture.transport.messageQueueForTest.isEmpty()) + val frameList = acceptWs.sentFrames.toList() + assertEquals( + listOf(fixture.rpc.encodeRequest(req1), fixture.rpc.encodeRequest(req2)), + frameList, + ) + } finally { + fixture.close() + } + } + + @Test + fun authClosingDoesNotPrematurelyDropPendingRpcAndClosedHaltsReconnect() = + kotlinx.coroutines.runBlocking { + val fixture = TestFixture() + try { + val fakeWs = FakeWebSocket(fixture.dummyRequest, accept = true, queueSizeValue = 0L) + fixture.transport.connectionGenerationForTest.set(1) + fixture.transport.intentionalCloseForTest.set(false) + + val listener = fixture.transport.createListenerForTest(1) + listener.onOpen(fakeWs, fixture.openResponse) + assertTrue(fixture.transport.isConnected) + + val pendingDeferred = fixture.rpc.request("gateway.ping", emptyMap()) + assertTrue(fixture.rpc.hasPendingCalls()) + + listener.onClosing(fakeWs, 4401, "AUTH_EXPIRED") + assertTrue(fixture.transport.isConnected) + assertTrue(fixture.rpc.hasPendingCalls()) + assertFalse(pendingDeferred.isCompleted) + + listener.onClosed(fakeWs, 4401, "AUTH_EXPIRED") + assertFalse(fixture.transport.isConnected) + assertNull(fixture.transport.reconnectJobForTest) + assertTrue(fixture.rpc.hasPendingCalls()) + assertFalse(pendingDeferred.isCompleted) + + fixture.transport.disconnect(clearPendingMessages = true) + assertFalse(pendingDeferred.isCompleted) + + pendingDeferred.cancel() + } finally { + fixture.close() + } + } +} diff --git a/app/src/main/java/com/m57/hermescontrol/data/ws/ConnectionHealth.kt b/app/src/main/java/com/m57/hermescontrol/data/ws/ConnectionHealth.kt new file mode 100644 index 00000000..89dcbbca --- /dev/null +++ b/app/src/main/java/com/m57/hermescontrol/data/ws/ConnectionHealth.kt @@ -0,0 +1,140 @@ +package com.m57.hermescontrol.data.ws + +import androidx.annotation.VisibleForTesting +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch + +internal class ConnectionHealth( + private val scope: CoroutineScope, + private val isConnected: () -> Boolean, + private val pingRequest: suspend (Long) -> Unit, + private val cancelSocket: () -> Unit, + private val pendingReply: () -> Boolean, + private val nowMs: () -> Long = { System.nanoTime() / 1_000_000L }, + private val log: (String) -> Unit = {}, + private val debugLog: (String) -> Unit = {}, +) { + companion object { + internal const val HEARTBEAT_INTERVAL_MS = 15_000L + internal const val STALE_THRESHOLD_MS = 30_000L + internal const val LIVENESS_PROBE_TIMEOUT_MS = 5_000L + } + + @Volatile + var lastPongTimestamp: Long = 0L + internal set + + private val _lastLatencyMs = MutableStateFlow(null) + val lastLatencyMs: StateFlow = _lastLatencyMs.asStateFlow() + + val isHealthy: Boolean + get() = isConnected() && (nowMs() - lastPongTimestamp < STALE_THRESHOLD_MS) + + private var healthJob: Job? = null + private var foregroundProbeJob: Job? = null + private var transportProbeJob: Job? = null + + fun start() { + healthJob?.cancel() + lastPongTimestamp = nowMs() + healthJob = + scope.launch { + while (isConnected()) { + delay(HEARTBEAT_INTERVAL_MS) + if (isConnected()) { + try { + ping(timeoutMs = LIVENESS_PROBE_TIMEOUT_MS) + } catch (e: Exception) { + log("Health check ping failed: ${e.message}") + } + } + if (!runHealthCheckPass()) break + } + } + } + + fun stop() { + healthJob?.cancel() + healthJob = null + _lastLatencyMs.value = null + } + + fun onInboundFrame() { + lastPongTimestamp = nowMs() + } + + suspend fun ping(timeoutMs: Long = LIVENESS_PROBE_TIMEOUT_MS): Long { + val start = nowMs() + pingRequest(timeoutMs) + val latency = (nowMs() - start).coerceAtLeast(0L) + lastPongTimestamp = nowMs() + _lastLatencyMs.value = latency + return latency + } + + @VisibleForTesting + internal fun setLastLatencyForTest(latency: Long?) { + _lastLatencyMs.value = latency + } + + private fun runHealthCheckPass(): Boolean { + if (!isConnected()) return false + val staleMs = nowMs() - lastPongTimestamp + if (staleMs <= STALE_THRESHOLD_MS) return true + log("WebSocket stale (${staleMs / 1000}s without frames) — cancelling to trigger reconnect") + cancelSocket() + return false + } + + @VisibleForTesting + internal fun forceHealthCheckForTest(staleMillis: Long) { + lastPongTimestamp = nowMs() - staleMillis + runHealthCheckPass() + } + + fun probeLivenessOnWake(deferredOnce: Boolean = false) { + if (!isConnected()) return + foregroundProbeJob?.cancel() + foregroundProbeJob = + scope.launch { + val alive = runCatching { ping(LIVENESS_PROBE_TIMEOUT_MS) }.isSuccess + if (alive) return@launch + if (pendingReply() && !deferredOnce) { + delay(3_000L) + probeLivenessOnWake(deferredOnce = true) + return@launch + } + log("Foreground liveness probe failed — cancelling socket") + cancelSocket() + } + } + + fun cancelForegroundProbe() { + foregroundProbeJob?.cancel() + foregroundProbeJob = null + } + + fun probeLivenessOnTransportChange( + timeoutMs: Long = LIVENESS_PROBE_TIMEOUT_MS, + onFailureAction: () -> Unit = cancelSocket, + ) { + if (!isConnected()) return + transportProbeJob?.cancel() + transportProbeJob = + scope.launch { + debugLog("Network transport changed — probing WebSocket liveness") + val alive = runCatching { ping(timeoutMs) }.isSuccess + if (alive) { + debugLog("WebSocket liveness probe succeeded on new transport") + return@launch + } + log("Transport change liveness probe failed — cancelling socket") + onFailureAction() + } + } +} diff --git a/app/src/main/java/com/m57/hermescontrol/data/ws/HermesWsClient.kt b/app/src/main/java/com/m57/hermescontrol/data/ws/HermesWsClient.kt index fe87906b..634af15e 100644 --- a/app/src/main/java/com/m57/hermescontrol/data/ws/HermesWsClient.kt +++ b/app/src/main/java/com/m57/hermescontrol/data/ws/HermesWsClient.kt @@ -2,30 +2,17 @@ package com.m57.hermescontrol.data.ws import android.util.Log import androidx.annotation.VisibleForTesting -import com.m57.hermescontrol.BuildConfig -import com.m57.hermescontrol.data.local.AuthManager -import com.m57.hermescontrol.data.remote.CookieManager -import com.m57.hermescontrol.data.remote.DashboardSessionTokenRefresher -import com.m57.hermescontrol.data.remote.NetworkMonitor import com.m57.hermescontrol.data.remote.OkHttpProvider -import com.m57.hermescontrol.data.remote.await import com.m57.hermescontrol.data.session.ActiveSessionHolder import com.m57.hermescontrol.data.ws.contract.ClientCapabilitiesParams -import com.m57.hermescontrol.data.ws.contract.PromptSubmitParams import com.m57.hermescontrol.data.ws.contract.RpcMethod import com.m57.hermescontrol.data.ws.contract.RpcMethods -import com.m57.hermescontrol.data.ws.contract.SessionCorrectionParams -import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceParams -import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job -import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.channels.BufferOverflow -import kotlinx.coroutines.currentCoroutineContext -import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharedFlow @@ -34,29 +21,14 @@ import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext -import kotlinx.serialization.decodeFromString -import kotlinx.serialization.encodeToString import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.buildJsonObject -import kotlinx.serialization.json.put -import okhttp3.Request -import okhttp3.RequestBody.Companion.toRequestBody -import okhttp3.Response import okhttp3.WebSocket import okhttp3.WebSocketListener -import okio.utf8Size -import java.util.Collections -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger -/** - * Connection status for the WebSocket client. - */ +/** Connection status for the WebSocket client. */ enum class ConnectionStatus { DISCONNECTED, CONNECTING, @@ -68,205 +40,16 @@ enum class ConnectionStatus { /** * WebSocket client for the Hermes Dashboard JSON-RPC 2.0 interface. - * - * Connects to `ws://HOST:PORT/api/ws?token=TOKEN`, auto-reconnects with - * exponential backoff, and emits parsed [WsEvent]s via [events] SharedFlow - * as well as direct callbacks. + * Facade coordinating transport, RPC correlation, replay tracking, and connection health. */ object HermesWsClient { private const val TAG = "HermesWsClient" - // ── Monotonic time helper (immune to NTP and wall-clock jumps) ───────── private fun monotonicTimeMs(): Long = System.nanoTime() / 1_000_000L - // ── Backoff settings (desktop parity: reconnect-backoff.ts full-jitter) ─ - - private var initialBackoffMs = 1_000L - private const val BACKOFF_BASE_MS = 300L - private const val BACKOFF_CAP_MS = 15_000L - private const val MAX_BACKOFF_MS = 30_000L - private const val BACKOFF_MULTIPLIER = 2.0 - private var testBackoffOverrideMs: Long? = null - - /** - * Test hook: the initial reconnect backoff, overridable so reconnect - * tests are deterministic instead of racing real time (CI runners - * routinely exceed fixed latch windows). Production keeps full-jitter backoff. - */ - @VisibleForTesting - internal fun setReconnectBackoffForTest(initialMillis: Long) { - initialBackoffMs = initialMillis - currentBackoff = initialMillis - testBackoffOverrideMs = initialMillis - } - - private fun nextBackoffDelayMs(): Long { - val override = testBackoffOverrideMs - if (override != null) return override - val ceiling = currentBackoff - if (ceiling <= 0L) return 0L - return (Math.random() * ceiling).toLong().coerceAtLeast(BACKOFF_BASE_MS).coerceAtMost(ceiling) - } - - private const val MAX_OUTBOUND_MESSAGE_BYTES = 16 * 1024 * 1024 - private const val OUTBOUND_DRAIN_TIMEOUT_MS = 60_000L - - /** - * Liveness thresholds aligned with reference clients (desktop/web): - * Ping every 15s using gateway.ping; 30s inbound silence deadline (issue #1165) - * as mobile secondary safety net for faster half-open socket recovery. - */ - private const val HEARTBEAT_INTERVAL_MS = 15_000L - private const val STALE_THRESHOLD_MS = 30_000L - private const val LIVENESS_PROBE_TIMEOUT_MS = 5_000L - - // ── Internal state (all access through synchronized / atomic) ──────── - - private val requestId = AtomicInteger(0) - private val capabilityRequestIds = ConcurrentHashMap.newKeySet() - private val connectionGeneration = AtomicInteger(0) - private val connected = AtomicBoolean(false) - private val intentionalClose = AtomicBoolean(false) - private val acceptQueuedMessages = AtomicBoolean(true) - private val appInForeground = AtomicBoolean(true) - private val externalActivityConnectionLease = AtomicBoolean(false) - private val backgroundConnectionLease = AtomicBoolean(false) - private val messageQueue = ConcurrentLinkedQueue() - private val queuedMessagesById = ConcurrentHashMap() private val outboundLock = Any() - - @Volatile - private var webSocket: WebSocket? = null - - @Volatile - private var closingSocket: WebSocket? = null - - @Volatile - private var currentBackoff = initialBackoffMs - private val wsScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - - /** - * Started lazily on the first [connect] instead of in [init]: the - * reconnect-on-network-change subscription only matters once a socket - * has actually been opened. init-time subscription also leaks a live - * real-IO collector into the unit-test JVM (class load), where - * NetworkMonitor emissions crashed it after unmockkAll. - */ - private val networkCollectorStarted = AtomicBoolean(false) - - @Volatile - private var reconnectJob: Job? = null - - @Volatile - private var outboundDrainJob: Job? = null - - // ── Sequence tracking & replay on reconnect (issue #1016) ───────────── - private val lastSeenSeq = ConcurrentHashMap() - - @Volatile - private var replayEpoch: String? = null - - @Volatile - private var replayInFlight: Boolean = false - private val replayHold = ConcurrentHashMap>>() - - @Volatile - private var replayJob: Job? = null - - // ── Health and Ping/Pong tracking ──────────────────────────────────── - - @Volatile - var lastPongTimestamp: Long = 0L - private set - - val isHealthy: Boolean - get() = isConnected && (monotonicTimeMs() - lastPongTimestamp < STALE_THRESHOLD_MS) - - // ── Observable latency tracking (issue #1017) ───────────────────────── - private val _lastLatencyMs = MutableStateFlow(null) - - /** Last measured round-trip latency in milliseconds via [ping], or null when disconnected / unmeasured. */ - val lastLatencyMs: StateFlow = _lastLatencyMs.asStateFlow() - - private var healthJob: Job? = null - - private fun startHealthTracking() { - healthJob?.cancel() - lastPongTimestamp = monotonicTimeMs() - healthJob = - wsScope.launch { - while (connected.get()) { - delay(HEARTBEAT_INTERVAL_MS) - if (connected.get()) { - try { - ping(timeoutMs = LIVENESS_PROBE_TIMEOUT_MS) - } catch (e: Exception) { - Log.w(TAG, "Health check ping failed: ${e.message}") - } - } - if (!runHealthCheckPass()) break - } - } - } - - /** - * Ultra-lightweight JSON-RPC liveness check and RTT measurement (issue #1017). - * Uses [WsMethods.GATEWAY_PING] which is answered inline in the WebSocket read - * loop (ws.py:350) and never queues behind worker dispatch pools. - * Updates [lastLatencyMs] and [lastPongTimestamp]. - */ - suspend fun ping(timeoutMs: Long = LIVENESS_PROBE_TIMEOUT_MS): Long { - val start = monotonicTimeMs() - request(WsMethods.GATEWAY_PING, emptyMap(), timeoutMs = timeoutMs).await() - val latency = (monotonicTimeMs() - start).coerceAtLeast(0L) - lastPongTimestamp = monotonicTimeMs() - _lastLatencyMs.value = latency - return latency - } - - @VisibleForTesting - internal fun setLastLatencyForTest(latency: Long?) { - _lastLatencyMs.value = latency - } - - /** - * One liveness pass. Returns false when the tracking loop should stop: - * either the connection is already gone or the watchdog fired a cancel. - * - * Cancelling (not close()) forces an immediate onFailure, which rides the - * existing teardown path: generation bump, RECONNECTING, scheduleReconnect. - */ - private fun runHealthCheckPass(): Boolean { - if (!connected.get()) return false - val staleMs = monotonicTimeMs() - lastPongTimestamp - if (staleMs <= STALE_THRESHOLD_MS) return true - Log.w(TAG, "WebSocket stale (${staleMs / 1000}s without frames) — cancelling to trigger reconnect") - synchronized(outboundLock) { - if (!connected.get()) return false - webSocket?.cancel() - } - return false - } - - /** - * Test hook: backdate liveness past the staleness threshold and force one - * synchronous watchdog pass, so reconnect tests are deterministic instead - * of racing the real 15s/45s cadence. - */ - @VisibleForTesting - internal fun forceHealthCheckForTest(staleMillis: Long) { - lastPongTimestamp = monotonicTimeMs() - staleMillis - runHealthCheckPass() - } - - private fun stopHealthTracking() { - healthJob?.cancel() - healthJob = null - _lastLatencyMs.value = null - } - - // ── Public observable stream ───────────────────────────────────────── + private val reconnectPolicy = ReconnectPolicy() private val parsedEvents = MutableSharedFlow( @@ -274,34 +57,79 @@ object HermesWsClient { onBufferOverflow = BufferOverflow.DROP_OLDEST, ) - /** Collect this from ViewModels to receive all parsed [WsEvent]s. */ val events: SharedFlow = parsedEvents.asSharedFlow() - // ── Connection status flow ────────────────────────────────────────── - private val _connectionStatus = MutableStateFlow(ConnectionStatus.DISCONNECTED) + private val connectionHealth = + ConnectionHealth( + scope = wsScope, + isConnected = { transport.isConnected }, + pingRequest = { timeoutMs -> + request(WsMethods.GATEWAY_PING, emptyMap(), timeoutMs = timeoutMs).await() + Unit + }, + cancelSocket = { transport.cancelSocket() }, + pendingReply = { transport.pendingReply }, + nowMs = ::monotonicTimeMs, + log = { Log.w(TAG, it) }, + debugLog = { Log.d(TAG, it) }, + ) - /** Observable connection status */ - val connectionStatus: StateFlow = _connectionStatus.asStateFlow() + private val replayTracker = + ReplayTracker( + scope = wsScope, + fetchEventsSince = { + call(RpcMethods.SESSION_EVENTS_SINCE, it, timeoutMs = 10_000L) + }, + emitEvent = { parsedEvents.tryEmit(it) }, + resolveStoredSessionId = { ActiveSessionHolder.resolveStoredSessionId(it) }, + onReplayFailure = { sid, e -> Log.w(TAG, "Replay failed for session $sid: ${e.message}") }, + ) - // ── Reply-pending tracking ───────────────────────────────────────── - // True while at least one agent turn is in flight. Drives the chat - // notification foreground service: it only runs while a reply is - // actually pending, instead of for the whole time the app is - // backgrounded (issue #794). - @Volatile - var pendingReply: Boolean = false + private val rpcChannel: RpcChannel = + RpcChannel( + scope = wsScope, + json = OkHttpProvider.json, + sendRequest = { method, params, onSent -> send(method, params, onSent) }, + sendFrame = { frameJson -> transport.sendDirectFrame(frameJson) }, + removeQueuedMessage = { transport.removeQueuedMessage(it) }, + onIdleCheck = { transport.disconnectIfIdleInBackground() }, + logRejected = { method, id -> Log.w(TAG, "Rejecting pending request on disconnect: $method (id=$id)") }, + outboundLock = outboundLock, + onResult = { id -> transport.onRpcResult(id) }, + onError = { id -> transport.onRpcError(id) }, + resolveStoredSessionId = { ActiveSessionHolder.resolveStoredSessionId(it) }, + logParseFailure = { e -> Log.e(TAG, "Failed to parse message: ${e.javaClass.simpleName}") }, + decorateParams = { method, params -> WsProfileParams.decorate(method, params) }, + ) + + private val transport: WsTransport = + WsTransport( + scope = wsScope, + outboundLock = outboundLock, + reconnectPolicy = reconnectPolicy, + connectionHealth = connectionHealth, + replayTracker = replayTracker, + rpcChannel = rpcChannel, + onRawMessage = { rawFrame -> + rpcChannel.handleIncomingFrame(rawFrame, replayTracker) { event -> parsedEvents.tryEmit(event) } + }, + ) + + val connectionStatus: StateFlow = transport.connectionStatus + + var pendingReply: Boolean by transport::pendingReply private set - private val pendingPromptSubmits = ConcurrentHashMap.newKeySet() + var lastPongTimestamp: Long by connectionHealth::lastPongTimestamp + private set + + val isHealthy: Boolean get() = connectionHealth.isHealthy + val lastLatencyMs: StateFlow = connectionHealth.lastLatencyMs + + suspend fun ping(timeoutMs: Long = ConnectionHealth.LIVENESS_PROBE_TIMEOUT_MS): Long = + connectionHealth.ping(timeoutMs) - // ── Credential warning (issue #534) ───────────────────────────────── - // Backend surfaces `credential_warning` in `gateway.ready` / `session.info` - // WS payloads (desktop `requestDesktopOnboarding`). Mobile has no equivalent - // at the auth layer, so we extract it here once, globally, and let any - // screen render a banner that deep-links to ProvidersScreen. private val _credentialWarning = MutableStateFlow(null) - - /** Non-null when the backend reports a credential warning to resolve. */ val credentialWarning: StateFlow = _credentialWarning.asStateFlow() fun clearCredentialWarning() { @@ -309,7 +137,9 @@ object HermesWsClient { } init { - // Extract credential_warning from gateway.ready / session.info payloads. + wsScope.launch { + parsedEvents.collect { transport.observeEvent(it) } + } wsScope.launch { events.collect { event -> val data: Map? = @@ -319,1326 +149,238 @@ object HermesWsClient { else -> null } val warning = data?.get("credential_warning") as? String - if (!warning.isNullOrBlank()) { - _credentialWarning.value = warning - } + if (!warning.isNullOrBlank()) _credentialWarning.value = warning } } - // Track whether a reply is actually in flight so the chat notification - // foreground service only runs while the user is waiting for one - // (issue #794), instead of for the whole time the app is backgrounded. - // Also covers turns started from other devices on the same session. wsScope.launch { - events.collect { event -> - when (event) { - is WsEvent.MessageStart, - is WsEvent.MessageToken, - is WsEvent.ThinkingDelta, - is WsEvent.ReasoningDelta, - is WsEvent.ToolStart, - -> { - pendingReply = true - } - - is WsEvent.MessageComplete -> { - pendingReply = false - disconnectIfIdleInBackground() - } - - else -> {} - } - } + events.filterIsInstance().collect { ChangeEventHub.emit(it) } } - // Forward parsed change events (issue #784) to the hub so screens can - // refresh on change without touching this singleton — its real static - // init must never run inside unit tests (see ChangeEventHub). - wsScope.launch { - events - .filterIsInstance() - .collect { ChangeEventHub.emit(it) } - } - // Track gateway replay_epoch across gateway.ready events (issue #1016) wsScope.launch { events.collect { event -> if (event is WsEvent.GatewayReady) { - // Advertise support for gateway server→client requests (issue #1197). - // Older gateways may reject this method; that is harmless. runCatching { send( RpcMethods.CLIENT_CAPABILITIES.name, - encodeParams( + rpcChannel.encodeParams( RpcMethods.CLIENT_CAPABILITIES, ClientCapabilitiesParams(serverRequests = true), ), - onSent = { id -> capabilityRequestIds.add(id) }, + onSent = { id -> rpcChannel.registerCapabilityRequest(id) }, queueIfDisconnected = false, ) } val epoch = event.data?.get("replay_epoch") as? String - if (!epoch.isNullOrEmpty()) { - if (replayEpoch != null && replayEpoch != epoch) { - lastSeenSeq.clear() - } - replayEpoch = epoch - } + replayTracker.updateEpoch(epoch) } } } } - // ── Connection helpers ──────────────────────────────────────────────── - - @VisibleForTesting - val isConnected: Boolean get() = connected.get() - - private var foregroundProbeJob: Job? = null - private var transportProbeJob: Job? = null - - fun setAppForeground(foreground: Boolean) { - appInForeground.set(foreground) - if (!foreground) { - disconnectIfIdleInBackground() - return - } - probeLivenessOnWake() - } - - private fun probeLivenessOnWake(deferredOnce: Boolean = false) { - if (!connected.get()) return - foregroundProbeJob?.cancel() - foregroundProbeJob = - wsScope.launch { - val alive = runCatching { ping(LIVENESS_PROBE_TIMEOUT_MS) }.isSuccess - if (alive) return@launch - if (pendingReply && !deferredOnce) { - delay(3_000L) - probeLivenessOnWake(deferredOnce = true) - return@launch - } - Log.w(TAG, "Foreground liveness probe failed — cancelling socket") - synchronized(outboundLock) { - if (connected.get()) webSocket?.cancel() - } - } - } - @VisibleForTesting - internal fun probeLivenessOnTransportChange( - timeoutMs: Long = LIVENESS_PROBE_TIMEOUT_MS, - onFailureAction: () -> Unit = { - synchronized(outboundLock) { - if (connected.get()) webSocket?.cancel() - } - }, - ) { - if (!connected.get()) return - transportProbeJob?.cancel() - transportProbeJob = - wsScope.launch { - Log.d(TAG, "Network transport changed — probing WebSocket liveness") - val alive = runCatching { ping(timeoutMs) }.isSuccess - if (alive) { - Log.d(TAG, "WebSocket liveness probe succeeded on new transport") - return@launch - } - Log.w(TAG, "Transport change liveness probe failed — cancelling socket") - onFailureAction() - } - } + val isConnected: Boolean get() = transport.isConnected - fun acquireExternalActivityConnectionLease() { - externalActivityConnectionLease.set(true) - } + fun setAppForeground(foreground: Boolean) = transport.setAppForeground(foreground) - fun releaseExternalActivityConnectionLease() { - externalActivityConnectionLease.set(false) - disconnectIfIdleInBackground() - } + fun acquireExternalActivityConnectionLease() = transport.acquireExternalActivityConnectionLease() - fun acquireBackgroundConnectionLease() { - backgroundConnectionLease.set(true) - } + fun releaseExternalActivityConnectionLease() = transport.releaseExternalActivityConnectionLease() - fun releaseBackgroundConnectionLease() { - backgroundConnectionLease.set(false) - disconnectIfIdleInBackground() - } + fun acquireBackgroundConnectionLease() = transport.acquireBackgroundConnectionLease() + + fun releaseBackgroundConnectionLease() = transport.releaseBackgroundConnectionLease() @VisibleForTesting - internal fun hasBackgroundConnectionLease(): Boolean = backgroundConnectionLease.get() + internal fun hasBackgroundConnectionLease(): Boolean = transport.hasBackgroundConnectionLease() - private fun disconnectIfIdleInBackground() { - synchronized(outboundLock) { - if (!appInForeground.get() && - !externalActivityConnectionLease.get() && - !backgroundConnectionLease.get() && - !pendingReply && - pendingCalls.isEmpty() && - messageQueue.isEmpty() - ) { - disconnect() - } - } - } + private fun openSocket() = transport.openSocket() - /** Open a WebSocket connection using settings from [AuthManager]. */ - fun connect() { - if (networkCollectorStarted.compareAndSet(false, true)) { - wsScope.launch { - NetworkMonitor.networkChanges.collect { networkAvailable -> - reconnectForNetworkChange(networkAvailable) - } - } - wsScope.launch { - NetworkMonitor.transportChanges.collect { - probeLivenessOnTransportChange() - } - } - } - var socketToCancel: WebSocket? = null - val shouldOpen = - synchronized(outboundLock) { - acceptQueuedMessages.set(true) - if (_connectionStatus.value == ConnectionStatus.AUTH_EXPIRED) { - connectionGeneration.incrementAndGet() - reconnectJob?.cancel() - reconnectJob = null - outboundDrainJob?.cancel() - outboundDrainJob = null - socketToCancel = webSocket - webSocket = null - closingSocket = null - connected.set(false) - stopHealthTracking() - } - if (connected.get()) { - Log.d(TAG, "Already connected — skipping") - return@synchronized false - } - if (_connectionStatus.value == ConnectionStatus.CONNECTING || - _connectionStatus.value == ConnectionStatus.RECONNECTING - ) { - Log.d(TAG, "Connection already in flight (${_connectionStatus.value}) — skipping") - return@synchronized false - } - // An explicit connect follows successful authentication and is - // therefore allowed to leave the terminal AUTH_EXPIRED state. - intentionalClose.set(false) - _connectionStatus.value = ConnectionStatus.CONNECTING - currentBackoff = initialBackoffMs - true - } - if (socketToCancel != null) { - rejectAllPending() - socketToCancel.cancel() - } - if (shouldOpen) openSocket() - } + fun connect() = transport.connect() - @VisibleForTesting - internal fun reconnectForNetworkChange( - networkAvailable: Boolean = NetworkMonitor.isConnected.value, + fun reconnectForNetworkChange( + networkAvailable: Boolean = com.m57.hermescontrol.data.remote.NetworkMonitor.isConnected.value, openReplacement: () -> Unit = ::openSocket, - ) { - val socketToCancel: WebSocket? - val shouldOpen: Boolean - synchronized(outboundLock) { - if (intentionalClose.get() || _connectionStatus.value == ConnectionStatus.AUTH_EXPIRED) { - return - } - Log.d(TAG, "Default network changed — reconnecting WebSocket") - currentBackoff = initialBackoffMs - reconnectJob?.cancel() - reconnectJob = null - outboundDrainJob?.cancel() - outboundDrainJob = null - connectionGeneration.incrementAndGet() - socketToCancel = webSocket - webSocket = null - closingSocket = null - connected.set(false) - stopHealthTracking() - shouldOpen = networkAvailable && AuthManager.isAutoReconnect() - _connectionStatus.value = - when { - !networkAvailable -> ConnectionStatus.NO_NETWORK - shouldOpen -> ConnectionStatus.RECONNECTING - else -> ConnectionStatus.DISCONNECTED - } - rejectAllPending() - } - socketToCancel?.cancel() - if (shouldOpen) openReplacement() - } + ) = transport.reconnectForNetworkChange(networkAvailable, openReplacement) - /** - * If a session cookie is present (gated mode), mint a fresh WS ticket - * from the dashboard. The ticket is single-use and has a 30-second TTL, - * so we must mint a new one on every connect (first launch and reconnect). - * - * The session cookie is attached automatically by the shared CookieJar on - * OkHttpProvider.probe (issue #470), so we no longer inject it manually. - * - * Returns true if the token is ready (either because we are not in gated mode, - * or because ticket refresh succeeded). Returns false if we are in gated mode - * and ticket refresh failed or cannot be performed. - */ - internal suspend fun refreshWsTicketIfNeeded(generation: Int): Boolean { - val isGated = - try { - AuthManager.serverStore.getLatestState().wsAuthParam == "ticket" - } catch (_: IllegalStateException) { - // serverStore not initialized yet (transient early call); treat - // as loopback so a stale-token refresh still runs. Log so a real - // misconfiguration isn't silently swallowed. - Log.w(TAG, "serverStore uninitialized during WS handshake; assuming non-gated") - false - } - if (!isGated) { - // The loopback dashboard token is regenerated on every server - // restart. Refresh it before each WebSocket handshake so automatic - // reconnect does not get stuck in AUTH_EXPIRED with a stale token. - val token = - runCatching { - DashboardSessionTokenRefresher.refreshAsync() - }.getOrNull() - synchronized(outboundLock) { - if (isCurrentGeneration(generation) && token != null) { - runCatching { AuthManager.setToken(token) } - } - } - return true - } + internal suspend fun refreshWsTicketIfNeeded(generation: Int): Boolean = + transport.refreshWsTicketIfNeeded(generation) - val ticketResult = requestWsTicket() - if (!ticketResult.ticket.isNullOrBlank()) { - synchronized(outboundLock) { - if (!isCurrentGeneration(generation)) return false - AuthManager.setToken(ticketResult.ticket) - } - if (BuildConfig.DEBUG) Log.d(TAG, "WS ticket refreshed") - return true - } - val status = - when { - ticketResult.exception -> ConnectionStatus.RECONNECTING - ticketResult.httpCode == 401 || ticketResult.httpCode == 403 -> ConnectionStatus.AUTH_EXPIRED - ticketResult.httpCode == 408 || ticketResult.httpCode == 429 -> ConnectionStatus.RECONNECTING - ticketResult.httpCode != null && ticketResult.httpCode >= 500 -> ConnectionStatus.RECONNECTING - else -> ConnectionStatus.DISCONNECTED - } - return handleWsTicketRefreshFailure(status, generation) - } + internal suspend fun mintWsTicket(): String? = transport.mintWsTicket() - /** - * Mint a fresh WS ticket for a *secondary* consumer (e.g. the kanban - * events stream) WITHOUT touching the shared token slot. - * - * The shared slot is a single-use ticket that the chat WebSocket consumes - * at handshake. If two clients mint and stash into the same slot on their - * own schedules, each can grab the other's just-consumed ticket and the - * pair starves each other into a permanent reconnect loop. Secondary - * consumers must use their own ticket per connection. - * - * Gated mode: cookie-auth'd POST to /api/auth/ws-ticket, returns the - * ticket. Loopback mode: refresh the dashboard token and return it. - * Returns null when a ticket could not be obtained. - */ - internal suspend fun mintWsTicket(): String? { - val isGated = - try { - AuthManager.serverStore.getLatestState().wsAuthParam == "ticket" - } catch (_: IllegalStateException) { - false - } - if (!isGated) { - DashboardSessionTokenRefresher.refreshAsync() - return AuthManager.getToken() - } - return requestWsTicket().ticket - } + fun disconnect(clearPendingMessages: Boolean = false) = transport.disconnect(clearPendingMessages) - private data class TicketRequestResult( - val ticket: String?, - val httpCode: Int?, - val exception: Boolean = false, - ) - - /** POST /api/auth/ws-ticket (cookie-auth'd via the shared CookieJar) and parse the ticket. */ - private suspend fun requestWsTicket(): TicketRequestResult { - try { - val client = OkHttpProvider.probe - val request = - Request - .Builder() - .url(AuthManager.endpointForBuild().resolve("api/auth/ws-ticket").toString()) - .post("{}".toRequestBody()) - .build() - - val (code, body) = - withContext(Dispatchers.IO) { - if (CookieManager.isInitialized()) { - CookieManager.useStore(CookieManager.cookieJar.currentServer()) - } - client.newCall(request).await().use { resp -> - resp.code to resp.body.string() - } - } - - if (code in 200..299) { - // Real JSON decode — the ticket is base64url and may contain - // characters a naive [^"]+ regex cannot extract (see - // testGatedMode_parsesEscapedTicketFromRealJsonShape). - val parsed = - runCatching { - OkHttpProvider.json.decodeFromString(body) - }.getOrNull() - val ticket = parsed?.ticket - if (!ticket.isNullOrBlank()) { - return TicketRequestResult(ticket, null) - } - Log.w(TAG, "WS ticket mint failed: unparseable ticket response") - } else { - Log.w(TAG, "WS ticket mint failed: HTTP $code") - return TicketRequestResult(null, code) - } - } catch (e: Exception) { - // Include the stack: the exception class alone (e.g. - // NetworkOnMainThreadException) can't show WHICH caller on what - // thread tripped the mint (main-scope kanban events connect vs - // the WS reconnect path). - Log.w(TAG, "WS ticket mint failed: ${e.javaClass.simpleName}", e) - return TicketRequestResult(null, null, exception = true) - } - return TicketRequestResult(null, null) - } - - private fun handleWsTicketRefreshFailure( - status: ConnectionStatus, - generation: Int, - ): Boolean { - synchronized(outboundLock) { - if (!isCurrentGeneration(generation)) return false - _connectionStatus.value = status - if (status == ConnectionStatus.RECONNECTING) scheduleReconnect() - } - return false - } - - /** Cleanly close the WebSocket and stop auto-reconnect. */ - fun disconnect(clearPendingMessages: Boolean = false) { - ActiveSessionHolder.clear() - synchronized(outboundLock) { - intentionalClose.set(true) - acceptQueuedMessages.set(!clearPendingMessages) - connectionGeneration.incrementAndGet() - reconnectJob?.cancel() - reconnectJob = null - outboundDrainJob?.cancel() - outboundDrainJob = null - foregroundProbeJob?.cancel() - foregroundProbeJob = null - stopHealthTracking() - webSocket?.close(1000, "Client closed") - webSocket = null - closingSocket = null - capabilityRequestIds.clear() - if (clearPendingMessages) { - backgroundConnectionLease.set(false) - messageQueue.clear() - queuedMessagesById.clear() - pendingPromptSubmits.clear() - pendingReply = false - lastSeenSeq.clear() - replayHold.clear() - replayEpoch = null - replayJob?.cancel() - replayJob = null - replayInFlight = false - } - connected.set(false) - _connectionStatus.value = ConnectionStatus.DISCONNECTED - } - } - - // ── Awaited RPC request layer (issue #526) ───────────────────────── - // Mirrors desktop apps/shared JsonRpcGatewayClient.request(): an in-flight - // map with a per-call timeout and rejectAllPending on socket close, so a - // dropped RPC response can't leave a caller awaiting forever. - - /** Thrown when an awaited [request] is neither answered nor rejected within [REQUEST_TIMEOUT_MS], or is rejected by a disconnect. */ class HermesRpcException( message: String, val code: Int = 0, val data: JsonElement? = null, ) : Exception(message) - /** - * Default per-request timeout. Matches the desktop - * `apps/shared` `JsonRpcGatewayClient.DEFAULT_REQUEST_TIMEOUT_MS` (120s), - * so legitimately long agent turns are not pruned early. - */ const val REQUEST_TIMEOUT_MS: Long = 120_000L - /** A single in-flight [request] awaiting its RPC result/error. */ - private data class PendingCall( - val method: String, - val deferred: CompletableDeferred, - var timeoutJob: Job? = null, - val suppressErrorEvent: Boolean = false, - ) - - /** Tracks in-flight [request] calls by their JSON-RPC id. */ - private val pendingCalls = ConcurrentHashMap() - - /** - * Send a JSON-RPC request that expects a result and returns a - * [CompletableDeferred] for it — mirroring the desktop - * `JsonRpcGatewayClient.request()`. Fire-and-forget notifications should - * keep using [send]. - * - * The deferred is completed on the matching [WsEvent.RpcResult] / - * [WsEvent.RpcError], rejected after [timeoutMs] (no response), or - * rejected by [rejectAllPending] when the socket closes. - * - * Untyped escape hatch, module-internal on purpose: gateway methods in the contract go through - * [call] / [requestTyped] with an `RpcMethods` descriptor. Only `gateway.ping` and tests use this. - */ internal fun request( method: String, params: Map = emptyMap(), timeoutMs: Long = REQUEST_TIMEOUT_MS, suppressErrorEvent: Boolean = false, - ): CompletableDeferred { - val deferred = CompletableDeferred() - val id = - send(method, params) { reqId -> - pendingCalls[reqId] = PendingCall(method, deferred, suppressErrorEvent = suppressErrorEvent) - } - deferred.invokeOnCompletion { cause -> - if (cause is CancellationException) { - pendingCalls.remove(id)?.let { call -> - call.timeoutJob?.cancel() - removeQueuedMessage(id) - disconnectIfIdleInBackground() - } - } - } - // Arm the per-request timeout (fires if the server never answers). - pendingCalls[id]?.timeoutJob = - wsScope.launch { - delay(timeoutMs) - resolvePending(id, null, JsonRpcError(-1, "Request timed out: $method")) - } - return deferred - } + ): CompletableDeferred = rpcChannel.request(method, params, timeoutMs, suppressErrorEvent) - /** - * Non-suspending typed twin of [request] for callers that must enqueue inside a - * non-suspending owner check and await the raw result themselves. - */ fun requestTyped( method: RpcMethod, params: P, timeoutMs: Long = REQUEST_TIMEOUT_MS, suppressErrorEvent: Boolean = false, - ): CompletableDeferred = request(method.name, encodeParams(method, params), timeoutMs, suppressErrorEvent) + ): CompletableDeferred = + request(method.name, rpcChannel.encodeParams(method, params), timeoutMs, suppressErrorEvent) - /** - * Send a typed JSON-RPC request and await its deserialized result. - * - * Params classes are strict writers; `profile` is a field ONLY on methods - * where callers pass an explicit override (e.g. session.create, session.resume); - * otherwise it is injected by WsProfileParams (an explicit value wins). - * Result classes are tolerant readers. - */ suspend fun call( method: RpcMethod, params: P, timeoutMs: Long = REQUEST_TIMEOUT_MS, suppressErrorEvent: Boolean = false, - ): R { - val deferred = requestTyped(method, params, timeoutMs, suppressErrorEvent) - val result = - try { - deferred.await() - } catch (e: CancellationException) { - deferred.cancel(e) - throw e - } - val element = result.toJsonElement() - return OkHttpProvider.json.decodeFromJsonElement(method.result, element) - } + ): R = rpcChannel.awaitResult(method, requestTyped(method, params, timeoutMs, suppressErrorEvent)) - private fun encodeParams( - method: RpcMethod, - params: P, - ): JsonObject { - val encoded = OkHttpProvider.json.encodeToJsonElement(method.params, params) - require(encoded is JsonObject) { - "RPC params for ${method.name} must serialize to a JsonObject, got ${encoded::class.simpleName}" - } - return encoded - } - - /** Complete (or fail) a single pending call and cancel its timer. */ - private fun resolvePending( - id: String, - result: Any?, - error: JsonRpcError?, - ) { - val call = pendingCalls.remove(id) ?: return - removeQueuedMessage(id) - call.timeoutJob?.cancel() - if (error != null) { - call.deferred.completeExceptionally( - HermesRpcException( - message = error.message, - code = error.code, - data = error.data, - ), - ) - } else { - call.deferred.complete(result) - } - disconnectIfIdleInBackground() - } - - /** - * Fail and clear every in-flight [request]. Called on disconnect / - * reconnect so callers awaiting a result don't hang across a socket - * close — mirrors desktop `JsonRpcGatewayClient.rejectAllPending(error)` - * invoked on socket close. - */ fun rejectAllPending( error: HermesRpcException = HermesRpcException("Connection lost — request cancelled"), - ) { - if (pendingCalls.isEmpty()) return - val snapshot = pendingCalls.toList() - pendingCalls.clear() - for ((id, call) in snapshot) { - removeQueuedMessage(id) - Log.w(TAG, "Rejecting pending request on disconnect: ${call.method} (id=$id)") - call.timeoutJob?.cancel() - call.deferred.completeExceptionally(error) - } - } + ) = rpcChannel.rejectAllPending(error) - /** - * Answer a gateway-originated JSON-RPC request. This deliberately bypasses - * [send]: server requests already have an id allocated by the gateway and - * responses must not contain a method or receive a new client id. - */ fun respondToServerRequest( id: String, result: JsonElement, - ): Boolean { - if (id.isBlank()) return false - val json = - buildJsonObject { - put("jsonrpc", "2.0") - put("id", id) - put("result", result) - }.toString() - synchronized(outboundLock) { - val ws = webSocket - if (ws == null || !connected.get()) return false - return ws.send(json) - } - } + ): Boolean = rpcChannel.respondToServerRequest(id, result) - /** Answer a gateway-originated request with a standard JSON-RPC error. */ fun respondToServerRequestError( id: String, code: Int, message: String, - ): Boolean { - if (id.isBlank()) return false - val json = - buildJsonObject { - put("jsonrpc", "2.0") - put("id", id) - put( - "error", - buildJsonObject { - put("code", code) - put("message", message) - }, - ) - }.toString() - synchronized(outboundLock) { - val ws = webSocket - if (ws == null || !connected.get()) return false - return ws.send(json) - } - } + ): Boolean = rpcChannel.respondToServerRequestError(id, code, message) - // ── Send helpers ───────────────────────────────────────────────────── - - /** - * Send a JSON-RPC request with the given [method] and optional [params]. - * - * Untyped escape hatch, module-internal on purpose: contract methods use the typed [send] overload. - * Kept for the legacy `*.respond` fallbacks (clarify, sudo, secret, vault) that have no contract entry. - * @return the request id used (can be matched against [WsEvent.RpcResult]). - */ internal fun send( method: String, params: Map = emptyMap(), onSent: ((String) -> Unit)? = null, - ): String = send(method, params, onSent, true) + ): String = send(method, params, onSent, queueIfDisconnected = true) - /** - * Send a typed fire-and-forget JSON-RPC request. - * - * Params classes are strict writers; `profile` is a field ONLY on methods - * where callers pass an explicit override (session.create, session.resume); - * otherwise it is injected by WsProfileParams (an explicit value wins). - * - * @return the request id used (can be matched against [WsEvent.RpcResult]). - */ fun send( method: RpcMethod, params: P, onSent: ((String) -> Unit)? = null, - ): String = send(method.name, encodeParams(method, params), onSent) + ): String = send(method.name, rpcChannel.encodeParams(method, params), onSent) private fun send( method: String, params: Map, onSent: ((String) -> Unit)?, queueIfDisconnected: Boolean, - ): String { - val id = requestId.incrementAndGet().toString() - val decoratedParams = WsProfileParams.decorate(method, params) - val request = - JsonRpcRequest( - id = id, - method = method, - params = decoratedParams.mapValues { it.value.toJsonElement() }, - ) - val json = OkHttpProvider.json.encodeToString(request) - if (BuildConfig.DEBUG) { - Log.d(TAG, formatSafeOutgoingFrameLog(id, method, request.params.keys, json.length, queued = false)) - } - var reconnect = false - synchronized(outboundLock) { - if (method == WsMethods.PROMPT_SUBMIT) { - pendingPromptSubmits.add(id) - pendingReply = true - } - val ws = webSocket - if (ws != null && connected.get()) { - // Never replay send(true): the server may have executed it even if its response is lost. - if (ws.send(json)) { - onSent?.invoke(id) - } else { - if (webSocket !== ws || !acceptQueuedMessages.get()) return@synchronized - if (queueIfDisconnected && isRetryableMessage(json)) { - onSent?.invoke(id) - Log.w(TAG, "WS rejected outgoing message — queuing for reconnect") - queueMessage(id, json) - recoverRejectedSocket(ws) - } else { - Log.w(TAG, "WS rejected oversized outgoing message — not retrying") - } - } - } else if (acceptQueuedMessages.get() && queueIfDisconnected) { - if (isRetryableMessage(json)) { - onSent?.invoke(id) - Log.d(TAG, "WS disconnected — queuing message") - queueMessage(id, json) - reconnect = true - } else { - Log.w(TAG, "WS disconnected with oversized outgoing message — not queueing") - } - } - } - if (reconnect) connect() - return id - } + ): String = + transport.send( + rpcChannel.createRequest(method, params), + onSent = onSent, + queueIfDisconnected = queueIfDisconnected, + ) - private fun isRetryableMessage(json: String): Boolean { - if (json.length > MAX_OUTBOUND_MESSAGE_BYTES) return false - if (json.length <= MAX_OUTBOUND_MESSAGE_BYTES / 4) return true - return json.utf8Size() <= MAX_OUTBOUND_MESSAGE_BYTES.toLong() - } - - private fun queueMessage( - id: String, - json: String, - ) { - if (queuedMessagesById.putIfAbsent(id, json) == null) messageQueue.add(json) - } - - private fun removeQueuedMessage(id: String) { - synchronized(outboundLock) { - queuedMessagesById.remove(id)?.let(messageQueue::remove) - } - } - - private fun markQueuedMessageSent(json: String) { - queuedMessagesById.entries.removeIf { it.value == json } - } - - private fun recoverRejectedSocket(ws: WebSocket) { - connected.set(false) - if (closingSocket === ws) return - _connectionStatus.value = ConnectionStatus.RECONNECTING - if (ws.queueSize() == 0L) { - ws.cancel() - scheduleReconnect() - return - } - if (intentionalClose.get()) return - outboundDrainJob?.cancel() - outboundDrainJob = - wsScope.launch { - delay(OUTBOUND_DRAIN_TIMEOUT_MS) - synchronized(outboundLock) { - if (!connected.get() && !intentionalClose.get() && webSocket === ws) { - ws.cancel() - outboundDrainJob = null - scheduleReconnect() - } - } - } - } - - /** Convenience: submit a user prompt to an existing session. */ fun sendMessage( sessionId: String, text: String, onSent: ((String) -> Unit)? = null, queued: Boolean = false, - ): String = - send( - method = RpcMethods.PROMPT_SUBMIT, - params = - PromptSubmitParams( - sessionId = sessionId, - text = text, - queued = queued.takeIf { it }, - ), - onSent = onSent, - ) + ): String = transport.sendMessage(sessionId, text, onSent, queued) - /** - * Convenience: redirect the active model turn while it is still generating - * (backend `session.redirect`). Fire-and-forget — the backend either rewrites - * the live turn, queues the correction as the next turn, or rejects it (the - * caller's ViewModel handles the `4010` fall-back to [sendMessage]). - */ fun sendRedirect( sessionId: String, text: String, onSent: ((String) -> Unit)? = null, - ): String = - send( - method = RpcMethods.SESSION_REDIRECT, - params = - SessionCorrectionParams( - sessionId = sessionId, - text = text, - ), - onSent = onSent, - ) - - // ── Sequence tracking & replay on reconnect (issue #1016) ───────────── - - private fun triggerReplay() { - // Arm the hold SYNCHRONOUSLY (still under onOpen's outboundLock) so a - // live frame racing the replay coroutine is parked instead of advancing - // lastSeenSeq past events the replay would then skip-drop (G2). - if (lastSeenSeq.isEmpty() || replayInFlight) return - replayInFlight = true - val sessionsToReplay = lastSeenSeq.keys().toList() - for (sid in sessionsToReplay) { - replayHold[sid] = Collections.synchronizedList(mutableListOf()) - } - replayJob?.cancel() - replayJob = - wsScope.launch { - fetchReplay(sessionsToReplay) - } - } - - private suspend fun fetchReplay(sessionsToReplay: List = lastSeenSeq.keys().toList()) { - try { - for (sid in sessionsToReplay) { - val lastSeen = lastSeenSeq[sid] ?: continue - try { - val res = - call( - RpcMethods.SESSION_EVENTS_SINCE, - SessionEventsSinceParams(sessionId = sid, lastSeen = lastSeen), - timeoutMs = 10_000L, - ) - - val epoch = res.epoch - val epochChanged = - !epoch.isNullOrEmpty() && replayEpoch != null && replayEpoch != epoch - val truncated = res.truncated == true - val latestSeq = res.latestSeq - - if (epochChanged) replayEpoch = epoch - - if (epochChanged || truncated) { - // Ring buffer (512 events) could not cover the gap or epoch changed (gateway restarted). - // Partial replay would silently hole the transcript — fast-forward watermark - // and request full transcript resync. - if (epochChanged) lastSeenSeq.clear() - if (latestSeq != null && !epochChanged) lastSeenSeq[sid] = latestSeq - parsedEvents.tryEmit(WsEvent.TranscriptResyncRequired(sid)) - continue - } - val eventsList = res.events - if (eventsList != null) { - for (element in eventsList) { - @Suppress("UNCHECKED_CAST") - val eventMap = (element as? JsonObject)?.toAny() as? Map ?: continue - val eventSeq = (eventMap["seq"] as? Number)?.toInt() - val eventSid = (eventMap["session_id"] as? String) ?: sid - if (eventSeq != null && eventSid.isNotEmpty()) { - val prev = lastSeenSeq[eventSid] ?: 0 - if (eventSeq <= prev) continue - lastSeenSeq[eventSid] = eventSeq - } - val parsedEvent = EventParser.parseParams(eventMap) - val finalEvent = - if (parsedEvent is WsEvent.MessageComplete) { - parsedEvent.copy( - storedSessionId = - ActiveSessionHolder.resolveStoredSessionId( - parsedEvent.sessionId, - ), - ) - } else { - parsedEvent - } - parsedEvents.tryEmit(finalEvent) - } - } - } catch (e: Exception) { - Log.w(TAG, "Replay failed for session $sid: ${e.message}") - parsedEvents.tryEmit(WsEvent.TranscriptResyncRequired(sid)) - } - } - } finally { - withContext(NonCancellable) { - for (sid in sessionsToReplay) { - val held = replayHold.remove(sid) - if (held != null) { - synchronized(held) { - for ((heldSeq, heldEvent) in held) { - if (heldSeq != null) { - val prev = lastSeenSeq[sid] ?: 0 - if (heldSeq <= prev) continue - lastSeenSeq[sid] = heldSeq - } - parsedEvents.tryEmit(heldEvent) - } - } - } - } - replayInFlight = false - } - } - } + ): String = transport.sendRedirect(sessionId, text, onSent) @VisibleForTesting - internal fun getSeqWatermarks(): Map = HashMap(lastSeenSeq) + internal fun getSeqWatermarks(): Map = replayTracker.getSeqWatermarks() @VisibleForTesting internal fun setSeqWatermark( sessionId: String, seq: Int, - ) { - lastSeenSeq[sessionId] = seq - } + ) = replayTracker.setSeqWatermark(sessionId, seq) @VisibleForTesting - internal fun clearSeqWatermarks() { - lastSeenSeq.clear() - replayHold.clear() - replayEpoch = null - replayInFlight = false - } + internal fun clearSeqWatermarks() = replayTracker.clearSeqWatermarks() @VisibleForTesting - internal fun isReplayInFlight(): Boolean = replayInFlight + internal fun isReplayInFlight(): Boolean = replayTracker.isReplayInFlight() @VisibleForTesting - internal fun setReplayEpochForTest(epoch: String?) { - replayEpoch = epoch - } + internal fun setReplayEpochForTest(epoch: String?) = replayTracker.setReplayEpochForTest(epoch) @VisibleForTesting - internal suspend fun fetchReplayForTest() { - fetchReplay() - } - - // ── Internal ───────────────────────────────────────────────────────── - - private fun openSocket() { - val generation = - synchronized(outboundLock) { - if (intentionalClose.get()) return - connectionGeneration.incrementAndGet() - } - wsScope.launch(Dispatchers.IO) { - if (!refreshWsTicketIfNeeded(generation)) { - Log.w(TAG, "Aborting openSocket: WS ticket refresh failed") - return@launch - } - if (!isCurrentGeneration(generation)) return@launch - val url = AuthManager.wsUrl() - val safeUrl = url.replace(Regex("token=[^&]+"), "token=REDACTED") - if (BuildConfig.DEBUG) Log.d(TAG, "Connecting to $safeUrl") - - val request = Request.Builder().url(url).build() - val newSocket = OkHttpProvider.websocket.newWebSocket(request, WsListenerImpl(generation)) - synchronized(outboundLock) { - if (connectionGeneration.get() == generation && !intentionalClose.get()) { - webSocket = newSocket - } else { - newSocket.cancel() - } - } - } - } - - private fun isCurrentGeneration(generation: Int): Boolean = - connectionGeneration.get() == generation && !intentionalClose.get() - - private fun scheduleReconnect() { - synchronized(outboundLock) { - if (intentionalClose.get() || - _connectionStatus.value == ConnectionStatus.AUTH_EXPIRED || - reconnectJob?.isActive == true - ) { - return - } - if (!AuthManager.isAutoReconnect()) { - if (BuildConfig.DEBUG) Log.d(TAG, "Auto-reconnect disabled") - _connectionStatus.value = ConnectionStatus.DISCONNECTED - return - } - if (!NetworkMonitor.isConnected.value) { - Log.d(TAG, "No network available — delaying reconnect scheduling") - _connectionStatus.value = ConnectionStatus.NO_NETWORK - return - } - val reconnectDelay = nextBackoffDelayMs() - currentBackoff = - (currentBackoff * BACKOFF_MULTIPLIER) - .toLong() - .coerceAtMost(MAX_BACKOFF_MS) - if (BuildConfig.DEBUG) Log.d(TAG, "Reconnecting in ${reconnectDelay}ms …") - - reconnectJob = - wsScope.launch { - delay(reconnectDelay) - val owner = currentCoroutineContext()[Job] - val shouldOpen = - synchronized(outboundLock) { - if (reconnectJob !== owner) { - false - } else { - reconnectJob = null - !intentionalClose.get() && - !connected.get() && - _connectionStatus.value != ConnectionStatus.AUTH_EXPIRED - } - } - if (shouldOpen) openSocket() - } - } - } - - private fun isTerminalAuthClose( - code: Int, - reason: String, - ): Boolean = - code == 4401 || code == 4403 || - reason.contains("unauthorized", ignoreCase = true) || - reason.startsWith("auth:", ignoreCase = true) - - // ── Listener ───────────────────────────────────────────────────────── - - private class WsListenerImpl( - private val generation: Int, - ) : WebSocketListener() { - private fun isCurrent(): Boolean = isCurrentGeneration(generation) - - private fun consumeCapabilityResponse(event: WsEvent): Boolean { - val id = - when (event) { - is WsEvent.RpcResult -> event.id - is WsEvent.RpcError -> event.id - else -> return false - } - // send() registers capability request IDs via onSent while holding - // outboundLock. Take the same lock here so a very fast gateway - // response cannot be consumed before its ID has been registered. - val isCapabilityResponse = - synchronized(outboundLock) { - capabilityRequestIds.remove(id) - } - if (!isCapabilityResponse) return false - - removeQueuedMessage(id) - return true - } - - override fun onOpen( - webSocket: WebSocket, - response: Response, - ) { - synchronized(outboundLock) { - if (!isCurrent()) { - webSocket.close(1000, "Superseded") - return - } - HermesWsClient.webSocket = webSocket - closingSocket = null - outboundDrainJob?.cancel() - outboundDrainJob = null - Log.i(TAG, "WebSocket opened") - connected.set(true) - _connectionStatus.value = ConnectionStatus.CONNECTED - currentBackoff = initialBackoffMs - startHealthTracking() - - while (true) { - val msg = messageQueue.peek() ?: break - if (!isRetryableMessage(msg)) { - Log.w(TAG, "Dropping oversized queued message") - messageQueue.poll() - markQueuedMessageSent(msg) - continue - } - if (BuildConfig.DEBUG) Log.d(TAG, formatSafeQueuedFrameLog(msg)) - if (!webSocket.send(msg)) { - recoverRejectedSocket(webSocket) - break - } - messageQueue.poll() - markQueuedMessageSent(msg) - } - triggerReplay() - } - disconnectIfIdleInBackground() - } - - override fun onMessage( - webSocket: WebSocket, - text: String, - ) { - if (!isCurrent() || HermesWsClient.webSocket !== webSocket) return - if (BuildConfig.DEBUG) Log.d(TAG, formatSafeIncomingFrameLog(text)) - lastPongTimestamp = monotonicTimeMs() - // Resolve any in-flight `request()` awaiting this RPC result/error - // (issue #526) before fanning the parsed event out to collectors. - val event = - try { - val rpc = OkHttpProvider.json.decodeFromString(text) - // Resolve pending request() calls with the RAW JsonElement from - // rpc.result, BEFORE EventParser.parse() converts it via toAny(). - // This preserves integer/float type fidelity needed by callers - // that re-deserialize the result into typed data classes. - val rpcId = rpc.id - if (rpcId != null && rpc.error == null && rpc.result != null) { - synchronized(outboundLock) { pendingPromptSubmits.remove(rpcId) } - removeQueuedMessage(rpcId) - resolvePending(rpcId, rpc.result, null) - } - - // Resume/replay responses carry still-open server requests - // separately from the event ring. Re-deliver them through the - // same generic path before exposing the RPC result. - @Suppress("UNCHECKED_CAST") - val openRequests = - (rpc.result?.toAny() as? Map<*, *>)?.get("open_requests") as? List> - openRequests?.forEach { openRequest -> - val openId = openRequest["id"] as? String ?: return@forEach - val method = openRequest["method"] as? String ?: return@forEach - - @Suppress("UNCHECKED_CAST") - val params = openRequest["params"] as? Map ?: emptyMap() - parsedEvents.tryEmit(WsEvent.ServerRequest(openId, method, params, replayed = true)) - } - - if (rpc.id == null) { - // Issue #1163: inspect scalars, not a throwaway copy of the whole params tree. - val sid = rpc.params?.eventSessionId() - val seq = ((rpc.params?.get("seq") as? JsonPrimitive)?.toAny() as? Number)?.toInt() - - if (!sid.isNullOrBlank() && seq != null) { - val prev = lastSeenSeq[sid] ?: 0 - if (seq <= prev) { - // Frame is already at or behind watermark — drop duplicate - return - } - val parsed = EventParser.parse(rpc, text) - val finalEvent = - if (parsed is WsEvent.MessageComplete) { - parsed.copy( - storedSessionId = ActiveSessionHolder.resolveStoredSessionId(parsed.sessionId), - ) - } else { - parsed - } - if (replayInFlight && replayHold.containsKey(sid)) { - replayHold[sid]?.add(seq to finalEvent) - return - } - lastSeenSeq[sid] = seq - parsedEvents.tryEmit(finalEvent) - return - } - } - - val parsed = EventParser.parse(rpc, text) - if (parsed is WsEvent.MessageComplete) { - parsed.copy( - storedSessionId = ActiveSessionHolder.resolveStoredSessionId(parsed.sessionId), - ) - } else { - parsed - } - } catch (e: Exception) { - Log.e(TAG, "Failed to parse message: ${e.javaClass.simpleName}") - WsEvent.Unknown(text) - } - if (consumeCapabilityResponse(event)) return - when (event) { - is WsEvent.RpcResult -> { - synchronized(outboundLock) { pendingPromptSubmits.remove(event.id) } - removeQueuedMessage(event.id) - resolvePending(event.id, event.result, null) - } - - is WsEvent.RpcError -> { - synchronized(outboundLock) { - if (pendingPromptSubmits.remove(event.id) && - pendingPromptSubmits.isEmpty() - ) { - pendingReply = false - disconnectIfIdleInBackground() - } - } - if (pendingCalls[event.id]?.suppressErrorEvent == true) { - // Opt-in suppression: the caller already handles the - // failure through the CompletableDeferred, so the event - // copy would only surface a duplicate UI banner for - // optional features (e.g. "subagent.list" on gateways - // without the method, issue #1089). Default request() - // behavior still emits the event for shared consumers. - resolvePending(event.id, null, event.error) - return - } - removeQueuedMessage(event.id) - resolvePending(event.id, null, event.error) - } - - else -> {} - } - // tryEmit on a DROP_OLDEST flow only returns false when the - // buffer is full AND no subscriber is draining; with extraBuffer=2048 - // and always-on init collectors this is unreachable in practice. - parsedEvents.tryEmit(event) - } - - override fun onClosing( - webSocket: WebSocket, - code: Int, - reason: String, - ) { - synchronized(outboundLock) { - if (!isCurrent()) { - webSocket.close(code, reason) - return - } - closingSocket = webSocket - // Do NOT log [reason] — it may carry server-side context. - Log.d(TAG, "WebSocket closing: $code") - if (isTerminalAuthClose(code, reason)) { - _connectionStatus.value = ConnectionStatus.AUTH_EXPIRED - } - webSocket.close(code, reason) - } - } - - override fun onClosed( - webSocket: WebSocket, - code: Int, - reason: String, - ) { - synchronized(outboundLock) { - if (!isCurrent()) return - connectionGeneration.incrementAndGet() - closingSocket = null - outboundDrainJob?.cancel() - outboundDrainJob = null - if (HermesWsClient.webSocket === webSocket) HermesWsClient.webSocket = null - // Do NOT log [reason] — it may carry server-side context. The - // reason is still inspected internally to detect auth failures. - Log.i(TAG, "WebSocket closed: $code") - connected.set(false) - ActiveSessionHolder.clear() - stopHealthTracking() - capabilityRequestIds.clear() - if (isTerminalAuthClose(code, reason)) { - _connectionStatus.value = ConnectionStatus.AUTH_EXPIRED - } else if (_connectionStatus.value != ConnectionStatus.AUTH_EXPIRED) { - _connectionStatus.value = ConnectionStatus.RECONNECTING - scheduleReconnect() - } - } - } - - override fun onFailure( - webSocket: WebSocket, - t: Throwable, - response: Response?, - ) { - synchronized(outboundLock) { - if (!isCurrent()) return - connectionGeneration.incrementAndGet() - closingSocket = null - outboundDrainJob?.cancel() - outboundDrainJob = null - if (HermesWsClient.webSocket === webSocket) HermesWsClient.webSocket = null - // Log the exception class only — [Throwable.message] can leak URLs - // or headers. The message is still inspected internally for auth - // detection. - Log.e(TAG, "WebSocket failure: ${t.javaClass.simpleName}", t) - connected.set(false) - ActiveSessionHolder.clear() - stopHealthTracking() - capabilityRequestIds.clear() - val code = response?.code ?: 0 - if (code == 401 || code == 4401 || code == 4403 || - t.message?.contains("401") == true || - t.message?.contains("4401") == true || - t.message?.contains("4403") == true || - t.message?.contains("unauthorized", ignoreCase = true) == true - ) { - _connectionStatus.value = ConnectionStatus.AUTH_EXPIRED - } else if (_connectionStatus.value != ConnectionStatus.AUTH_EXPIRED) { - _connectionStatus.value = ConnectionStatus.RECONNECTING - scheduleReconnect() - } - } - } - } + internal suspend fun fetchReplayForTest() = replayTracker.fetchReplayForTest() @VisibleForTesting - internal fun setConnectionStatusForTest(status: ConnectionStatus) { - _connectionStatus.value = status - } + internal fun probeLivenessOnTransportChange( + timeoutMs: Long = ConnectionHealth.LIVENESS_PROBE_TIMEOUT_MS, + onFailureAction: () -> Unit = { transport.cancelSocket() }, + ) = connectionHealth.probeLivenessOnTransportChange(timeoutMs, onFailureAction) + + @VisibleForTesting + internal fun setLastLatencyForTest(latency: Long?) = connectionHealth.setLastLatencyForTest(latency) + + @VisibleForTesting + internal fun forceHealthCheckForTest(staleMillis: Long) = connectionHealth.forceHealthCheckForTest(staleMillis) + + @VisibleForTesting + internal fun setReconnectBackoffForTest(initialMillis: Long) = transport.setReconnectBackoffForTest(initialMillis) + + @VisibleForTesting + internal fun setConnectionStatusForTest(status: ConnectionStatus) = transport.setConnectionStatusForTest(status) + + @VisibleForTesting + internal val connectedForTest: AtomicBoolean + get() = transport.connectedForTest + + @VisibleForTesting + internal val intentionalCloseForTest: AtomicBoolean + get() = transport.intentionalCloseForTest + + @VisibleForTesting + internal val connectionGenerationForTest: AtomicInteger + get() = transport.connectionGenerationForTest + + @VisibleForTesting + internal val messageQueueForTest: ConcurrentLinkedQueue + get() = transport.messageQueueForTest + + @VisibleForTesting + internal val outboundLockForTest: Any + get() = transport.outboundLockForTest + + @VisibleForTesting + internal val acceptQueuedMessagesForTest: AtomicBoolean + get() = transport.acceptQueuedMessagesForTest + + @VisibleForTesting + internal val connectionStatusForTest: MutableStateFlow + get() = transport.connectionStatusForTest + + @VisibleForTesting + internal var webSocketForTest: WebSocket? by transport::webSocketForTest + + @VisibleForTesting + internal var currentBackoffForTest: Long by transport::currentBackoffForTest + + @VisibleForTesting + internal var reconnectJobForTest: Job? by transport::reconnectJobForTest + + @VisibleForTesting + internal var outboundDrainJobForTest: Job? by transport::outboundDrainJobForTest + + @VisibleForTesting + internal fun scheduleReconnectForTest() = transport.scheduleReconnectForTest() + + @VisibleForTesting + internal fun createListenerForTest(generation: Int): WebSocketListener = transport.createListenerForTest(generation) + + @VisibleForTesting + internal fun pendingCallIdsForTest(): Set = rpcChannel.pendingCallIdsForTest() + + @VisibleForTesting + internal fun pendingCallTimeoutForTest(id: String): Job? = rpcChannel.pendingCallTimeoutForTest(id) @VisibleForTesting internal fun formatSafeOutgoingFrameLog( @@ -1647,47 +389,11 @@ object HermesWsClient { paramsKeys: Collection?, byteLength: Int, queued: Boolean = false, - ): String { - val prefix = if (queued) "→ (queued)" else "→" - val keys = paramsKeys?.joinToString(",", prefix = "[", postfix = "]") ?: "[]" - return "$prefix id=$id method=$method paramsKeys=$keys bytes=$byteLength" - } + ): String = transport.formatSafeOutgoingFrameLog(id, method, paramsKeys, byteLength, queued) @VisibleForTesting - internal fun formatSafeQueuedFrameLog(msg: String): String = - try { - val req = OkHttpProvider.json.decodeFromString(msg) - formatSafeOutgoingFrameLog(req.id, req.method, req.params.keys, msg.length, queued = true) - } catch (_: Throwable) { - "→ (queued) frame bytes=${msg.length}" - } + internal fun formatSafeQueuedFrameLog(msg: String): String = transport.formatSafeQueuedFrameLog(msg) @VisibleForTesting - internal fun formatSafeIncomingFrameLog(text: String): String = - try { - val element = OkHttpProvider.json.parseToJsonElement(text) - if (element is JsonObject) { - val id = (element["id"] as? JsonPrimitive)?.content - val method = (element["method"] as? JsonPrimitive)?.content - val hasResult = element.containsKey("result") - val resultKeys = - (element["result"] as? JsonObject)?.keys?.joinToString( - ",", - prefix = "[", - postfix = "]", - ) - val errorObj = element["error"] as? JsonObject - val errorCode = (errorObj?.get("code") as? JsonPrimitive)?.content - val sb = StringBuilder("← ") - if (id != null) sb.append("id=").append(id).append(" ") - if (method != null) sb.append("method=").append(method).append(" ") - if (hasResult) sb.append("result=").append(resultKeys ?: "present").append(" ") - if (errorCode != null) sb.append("errorCode=").append(errorCode).append(" ") - sb.append("bytes=").append(text.length).toString() - } else { - "← non-object frame bytes=${text.length}" - } - } catch (_: Throwable) { - "← frame bytes=${text.length}" - } + internal fun formatSafeIncomingFrameLog(text: String): String = transport.formatSafeIncomingFrameLog(text) } diff --git a/app/src/main/java/com/m57/hermescontrol/data/ws/ReconnectPolicy.kt b/app/src/main/java/com/m57/hermescontrol/data/ws/ReconnectPolicy.kt new file mode 100644 index 00000000..ae7fa4e4 --- /dev/null +++ b/app/src/main/java/com/m57/hermescontrol/data/ws/ReconnectPolicy.kt @@ -0,0 +1,39 @@ +package com.m57.hermescontrol.data.ws + +internal class ReconnectPolicy( + private val random: () -> Double = { Math.random() }, +) { + @Volatile + var currentBackoff: Long = INITIAL_BACKOFF_MS + private var initialBackoffMs: Long = INITIAL_BACKOFF_MS + private var testBackoffOverrideMs: Long? = null + + fun nextDelayMs(): Long { + val override = testBackoffOverrideMs + if (override != null) return override + val ceiling = currentBackoff + if (ceiling <= 0L) return 0L + return (random() * ceiling).toLong().coerceAtLeast(BACKOFF_BASE_MS).coerceAtMost(ceiling) + } + + fun advance() { + currentBackoff = (currentBackoff * BACKOFF_MULTIPLIER).toLong().coerceAtMost(MAX_BACKOFF_MS) + } + + fun reset() { + currentBackoff = initialBackoffMs + } + + fun setBackoffForTest(initialMillis: Long) { + initialBackoffMs = initialMillis + currentBackoff = initialMillis + testBackoffOverrideMs = initialMillis + } + + companion object { + private const val INITIAL_BACKOFF_MS = 1_000L + private const val BACKOFF_BASE_MS = 300L + private const val MAX_BACKOFF_MS = 30_000L + private const val BACKOFF_MULTIPLIER = 2.0 + } +} diff --git a/app/src/main/java/com/m57/hermescontrol/data/ws/ReplayTracker.kt b/app/src/main/java/com/m57/hermescontrol/data/ws/ReplayTracker.kt new file mode 100644 index 00000000..7cede6ac --- /dev/null +++ b/app/src/main/java/com/m57/hermescontrol/data/ws/ReplayTracker.kt @@ -0,0 +1,245 @@ +package com.m57.hermescontrol.data.ws + +import androidx.annotation.VisibleForTesting +import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceParams +import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.JsonObject +import java.util.Collections +import java.util.concurrent.ConcurrentHashMap + +/** + * Tracks session event sequence watermarks and orchestrates event replay on reconnect. + * + * Extracted from [HermesWsClient] (issue #1016, issue #1376). + */ +internal class ReplayTracker( + private val scope: CoroutineScope, + private val fetchEventsSince: suspend (SessionEventsSinceParams) -> SessionEventsSinceResult, + private val emitEvent: (WsEvent) -> Unit, + private val resolveStoredSessionId: (String?) -> String? = { it }, + private val onReplayFailure: (String, Exception) -> Unit = { _, _ -> }, +) { + private val lastSeenSeq = ConcurrentHashMap() + + @Volatile + private var replayEpoch: String? = null + + @Volatile + private var replayInFlight: Boolean = false + + private val replayHold = ConcurrentHashMap>>() + + @Volatile + private var replayJob: Job? = null + + /** + * Synchronously arms the hold buffers for all known sessions and launches replay in [scope]. + * Arming synchronously ensures racing live frames are held instead of advancing watermarks. + */ + fun triggerReplay() { + if (lastSeenSeq.isEmpty() || replayInFlight) return + replayInFlight = true + val sessionsToReplay = lastSeenSeq.keys().toList() + for (sid in sessionsToReplay) { + replayHold[sid] = Collections.synchronizedList(mutableListOf()) + } + replayJob?.cancel() + replayJob = + scope.launch { + fetchReplay(sessionsToReplay) + } + } + + /** + * Updates replay epoch from gateway.ready. + * When [epoch] is non-empty and changes from a previous non-null value, resets sequence watermarks. + */ + fun updateEpoch(epoch: String?) { + if (!epoch.isNullOrEmpty()) { + if (replayEpoch != null && replayEpoch != epoch) { + lastSeenSeq.clear() + } + replayEpoch = epoch + } + } + + /** + * Checks if an incoming event is a duplicate against the current watermark for [sessionId]. + * Returns true if [seq] <= watermark. + */ + fun isDuplicate( + sessionId: String, + seq: Int, + ): Boolean { + val prev = lastSeenSeq[sessionId] ?: 0 + return seq <= prev + } + + /** + * Processes an incoming live event with sequence tracking. + * Evaluates [eventFactory] only after checking for duplicates. + * If duplicate, drops it without calling [eventFactory]. + * If replay is in flight for [sessionId], buffers it in replayHold. + * Otherwise advances watermark and emits the event. + */ + fun acceptLiveEvent( + sessionId: String, + seq: Int, + eventFactory: () -> WsEvent, + ) { + if (isDuplicate(sessionId, seq)) { + return + } + val finalEvent = decorateEvent(eventFactory()) + if (replayInFlight && replayHold.containsKey(sessionId)) { + replayHold[sessionId]?.add(seq to finalEvent) + return + } + lastSeenSeq[sessionId] = seq + emitEvent(finalEvent) + } + + /** + * Processes an incoming live event with sequence tracking. + * If duplicate, drops it. + * If replay is in flight for [sessionId], buffers it in replayHold. + * Otherwise advances watermark and emits the event. + */ + fun acceptLiveEvent( + sessionId: String, + seq: Int, + event: WsEvent, + ) { + acceptLiveEvent(sessionId, seq) { event } + } + + private fun decorateEvent(event: WsEvent): WsEvent = + if (event is WsEvent.MessageComplete) { + event.copy( + storedSessionId = resolveStoredSessionId(event.sessionId), + ) + } else { + event + } + + private suspend fun fetchReplay(sessionsToReplay: List = lastSeenSeq.keys().toList()) { + try { + for (sid in sessionsToReplay) { + val lastSeen = lastSeenSeq[sid] ?: continue + try { + val res = + fetchEventsSince( + SessionEventsSinceParams(sessionId = sid, lastSeen = lastSeen), + ) + + val epoch = res.epoch + val epochChanged = + !epoch.isNullOrEmpty() && replayEpoch != null && replayEpoch != epoch + val truncated = res.truncated == true + val latestSeq = res.latestSeq + + if (epochChanged) replayEpoch = epoch + + if (epochChanged || truncated) { + // Ring buffer (512 events) could not cover the gap or epoch changed (gateway restarted). + // Partial replay would silently hole the transcript — fast-forward watermark + // and request full transcript resync. + if (epochChanged) lastSeenSeq.clear() + if (latestSeq != null && !epochChanged) lastSeenSeq[sid] = latestSeq + emitEvent(WsEvent.TranscriptResyncRequired(sid)) + continue + } + val eventsList = res.events + if (eventsList != null) { + for (element in eventsList) { + @Suppress("UNCHECKED_CAST") + val eventMap = (element as? JsonObject)?.toAny() as? Map ?: continue + val eventSeq = (eventMap["seq"] as? Number)?.toInt() + val eventSid = (eventMap["session_id"] as? String) ?: sid + if (eventSeq != null && eventSid.isNotEmpty()) { + val prev = lastSeenSeq[eventSid] ?: 0 + if (eventSeq <= prev) continue + lastSeenSeq[eventSid] = eventSeq + } + val parsedEvent = EventParser.parseParams(eventMap) + val finalEvent = decorateEvent(parsedEvent) + emitEvent(finalEvent) + } + } + } catch (e: Exception) { + onReplayFailure(sid, e) + emitEvent(WsEvent.TranscriptResyncRequired(sid)) + } + } + } finally { + withContext(NonCancellable) { + for (sid in sessionsToReplay) { + val held = replayHold.remove(sid) + if (held != null) { + synchronized(held) { + for ((heldSeq, heldEvent) in held) { + if (heldSeq != null) { + val prev = lastSeenSeq[sid] ?: 0 + if (heldSeq <= prev) continue + lastSeenSeq[sid] = heldSeq + } + emitEvent(heldEvent) + } + } + } + } + replayInFlight = false + } + } + } + + /** + * Complete teardown matching disconnect (issue #1016). + * Cancels any active replay job and clears all holds, watermarks, and epoch. + */ + fun clear() { + lastSeenSeq.clear() + replayHold.clear() + replayEpoch = null + replayJob?.cancel() + replayJob = null + replayInFlight = false + } + + @VisibleForTesting + internal fun getSeqWatermarks(): Map = HashMap(lastSeenSeq) + + @VisibleForTesting + internal fun setSeqWatermark( + sessionId: String, + seq: Int, + ) { + lastSeenSeq[sessionId] = seq + } + + @VisibleForTesting + internal fun clearSeqWatermarks() { + lastSeenSeq.clear() + replayHold.clear() + replayEpoch = null + replayInFlight = false + } + + @VisibleForTesting + internal fun isReplayInFlight(): Boolean = replayInFlight + + @VisibleForTesting + internal fun setReplayEpochForTest(epoch: String?) { + replayEpoch = epoch + } + + @VisibleForTesting + internal suspend fun fetchReplayForTest() { + fetchReplay() + } +} diff --git a/app/src/main/java/com/m57/hermescontrol/data/ws/RpcChannel.kt b/app/src/main/java/com/m57/hermescontrol/data/ws/RpcChannel.kt new file mode 100644 index 00000000..6f8cfa5d --- /dev/null +++ b/app/src/main/java/com/m57/hermescontrol/data/ws/RpcChannel.kt @@ -0,0 +1,353 @@ +package com.m57.hermescontrol.data.ws + +import com.m57.hermescontrol.data.ws.contract.RpcMethod +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +/** + * Socket-independent JSON-RPC 2.0 request correlation and response dispatcher. + * Manages request IDs, in-flight pending calls, timeouts, cancellation cleanup, + * and server response frame formatting. + */ +internal class RpcChannel( + private val scope: CoroutineScope, + private val json: Json, + private val sendRequest: (method: String, params: Map, onSent: ((String) -> Unit)?) -> String, + private val sendFrame: (String) -> Boolean, + private val removeQueuedMessage: (String) -> Unit, + private val onIdleCheck: () -> Unit, + private val logRejected: (method: String, id: String) -> Unit = { _, _ -> }, + private val outboundLock: Any = Any(), + private val onResult: (String) -> Unit = {}, + private val onError: (String) -> Unit = {}, + private val resolveStoredSessionId: (String?) -> String? = { it }, + private val logParseFailure: (Exception) -> Unit = {}, + private val decorateParams: (String, Map) -> Map = { _, p -> p }, +) { + /** A single in-flight [request] awaiting its RPC result/error. */ + private data class PendingCall( + val method: String, + val deferred: CompletableDeferred, + var timeoutJob: Job? = null, + val suppressErrorEvent: Boolean = false, + ) + + private val requestId = AtomicInteger(0) + private val pendingCalls = ConcurrentHashMap() + private val capabilityRequestIds = ConcurrentHashMap.newKeySet() + + fun nextId(): String = requestId.incrementAndGet().toString() + + fun hasPendingCalls(): Boolean = pendingCalls.isNotEmpty() + + fun pendingCallIdsForTest(): Set = pendingCalls.keys.toSet() + + fun pendingCallTimeoutForTest(id: String): Job? = pendingCalls[id]?.timeoutJob + + fun isErrorEventSuppressed(id: String): Boolean = pendingCalls[id]?.suppressErrorEvent == true + + fun registerCapabilityRequest(id: String) { + capabilityRequestIds.add(id) + } + + fun clearCapabilityRequests() { + capabilityRequestIds.clear() + } + + fun consumeCapabilityResponse(event: WsEvent): Boolean { + val id = + when (event) { + is WsEvent.RpcResult -> event.id + is WsEvent.RpcError -> event.id + else -> return false + } + val isCapabilityResponse = + synchronized(outboundLock) { + capabilityRequestIds.remove(id) + } + if (!isCapabilityResponse) return false + + removeQueuedMessage(id) + return true + } + + fun createRequest( + method: String, + params: Map, + ): JsonRpcRequest { + val id = nextId() + val decorated = decorateParams(method, params) + return JsonRpcRequest( + id = id, + method = method, + params = decorated.mapValues { it.value.toJsonElement() }, + ) + } + + fun encodeRequest(request: JsonRpcRequest): String = json.encodeToString(request) + + /** + * Send a JSON-RPC request that expects a result and returns a [CompletableDeferred] for it. + */ + fun request( + method: String, + params: Map = emptyMap(), + timeoutMs: Long = HermesWsClient.REQUEST_TIMEOUT_MS, + suppressErrorEvent: Boolean = false, + ): CompletableDeferred { + val deferred = CompletableDeferred() + val id = + sendRequest(method, params) { reqId -> + pendingCalls[reqId] = PendingCall(method, deferred, suppressErrorEvent = suppressErrorEvent) + } + deferred.invokeOnCompletion { cause -> + if (cause is CancellationException) { + pendingCalls.remove(id)?.let { call -> + call.timeoutJob?.cancel() + removeQueuedMessage(id) + onIdleCheck() + } + } + } + // Arm the per-request timeout (fires if the server never answers). + pendingCalls[id]?.timeoutJob = + scope.launch { + delay(timeoutMs) + resolvePending(id, null, JsonRpcError(-1, "Request timed out: $method")) + } + return deferred + } + + /** + * Non-suspending typed twin of [request] for callers that must enqueue inside a + * non-suspending owner check and await the raw result themselves. + */ + fun requestTyped( + method: RpcMethod, + params: P, + timeoutMs: Long = HermesWsClient.REQUEST_TIMEOUT_MS, + suppressErrorEvent: Boolean = false, + ): CompletableDeferred = request(method.name, encodeParams(method, params), timeoutMs, suppressErrorEvent) + + /** + * Send a typed JSON-RPC request and await its deserialized result. + */ + suspend fun call( + method: RpcMethod, + params: P, + timeoutMs: Long = HermesWsClient.REQUEST_TIMEOUT_MS, + suppressErrorEvent: Boolean = false, + ): R = awaitResult(method, requestTyped(method, params, timeoutMs, suppressErrorEvent)) + + suspend fun awaitResult( + method: RpcMethod, + deferred: CompletableDeferred, + ): R { + val result = + try { + deferred.await() + } catch (e: CancellationException) { + deferred.cancel(e) + throw e + } + val element = result.toJsonElement() + return json.decodeFromJsonElement(method.result, element) + } + + fun encodeParams( + method: RpcMethod, + params: P, + ): JsonObject { + val encoded = json.encodeToJsonElement(method.params, params) + require(encoded is JsonObject) { + "RPC params for ${method.name} must serialize to a JsonObject, got ${encoded::class.simpleName}" + } + return encoded + } + + /** Complete (or fail) a single pending call and cancel its timer. */ + fun resolvePending( + id: String, + result: Any?, + error: JsonRpcError?, + ) { + val call = pendingCalls.remove(id) ?: return + removeQueuedMessage(id) + call.timeoutJob?.cancel() + if (error != null) { + call.deferred.completeExceptionally( + HermesWsClient.HermesRpcException( + message = error.message, + code = error.code, + data = error.data, + ), + ) + } else { + call.deferred.complete(result) + } + onIdleCheck() + } + + /** + * Fail and clear every in-flight [request]. + */ + fun rejectAllPending( + error: HermesWsClient.HermesRpcException = + HermesWsClient.HermesRpcException("Connection lost — request cancelled"), + ) { + if (pendingCalls.isEmpty()) return + val snapshot = pendingCalls.toList() + pendingCalls.clear() + for ((id, call) in snapshot) { + removeQueuedMessage(id) + logRejected(call.method, id) + call.timeoutJob?.cancel() + call.deferred.completeExceptionally(error) + } + } + + /** + * Answer a gateway-originated JSON-RPC request. + */ + fun respondToServerRequest( + id: String, + result: JsonElement, + ): Boolean { + if (id.isBlank()) return false + val frameJson = + buildJsonObject { + put("jsonrpc", "2.0") + put("id", id) + put("result", result) + }.toString() + return sendFrame(frameJson) + } + + /** Answer a gateway-originated request with a standard JSON-RPC error. */ + fun respondToServerRequestError( + id: String, + code: Int, + message: String, + ): Boolean { + if (id.isBlank()) return false + val frameJson = + buildJsonObject { + put("jsonrpc", "2.0") + put("id", id) + put( + "error", + buildJsonObject { + put("code", code) + put("message", message) + }, + ) + }.toString() + return sendFrame(frameJson) + } + + /** + * Handles an incoming WebSocket text frame: JSON decoding, request correlation, + * server request dispatch, sequence deduplication via [replayTracker], and event emission. + */ + fun handleIncomingFrame( + text: String, + replayTracker: ReplayTracker, + emitEvent: (WsEvent) -> Unit, + ) { + val event = + try { + val rpc = json.decodeFromString(text) + // Resolve pending request() calls with the RAW JsonElement from + // rpc.result, BEFORE EventParser.parse() converts it via toAny(). + // This preserves integer/float type fidelity needed by callers + // that re-deserialize the result into typed data classes. + val rpcId = rpc.id + if (rpcId != null && rpc.error == null && rpc.result != null) { + onResult(rpcId) + removeQueuedMessage(rpcId) + resolvePending(rpcId, rpc.result, null) + } + + // Resume/replay responses carry still-open server requests + // separately from the event ring. Re-deliver them through the + // same generic path before exposing the RPC result. + @Suppress("UNCHECKED_CAST") + val openRequests = + (rpc.result?.toAny() as? Map<*, *>)?.get("open_requests") as? List> + openRequests?.forEach { openRequest -> + val openId = openRequest["id"] as? String ?: return@forEach + val method = openRequest["method"] as? String ?: return@forEach + + @Suppress("UNCHECKED_CAST") + val params = openRequest["params"] as? Map ?: emptyMap() + emitEvent(WsEvent.ServerRequest(openId, method, params, replayed = true)) + } + + if (rpc.id == null) { + // Issue #1163: inspect scalars, not a throwaway copy of the whole params tree. + val sid = rpc.params?.eventSessionId() + val seq = ((rpc.params?.get("seq") as? JsonPrimitive)?.toAny() as? Number)?.toInt() + + if (!sid.isNullOrBlank() && seq != null) { + replayTracker.acceptLiveEvent(sid, seq) { + EventParser.parse(rpc, text) + } + return + } + } + + val parsed = EventParser.parse(rpc, text) + if (parsed is WsEvent.MessageComplete) { + parsed.copy( + storedSessionId = resolveStoredSessionId(parsed.sessionId), + ) + } else { + parsed + } + } catch (e: Exception) { + logParseFailure(e) + WsEvent.Unknown(text) + } + if (consumeCapabilityResponse(event)) return + when (event) { + is WsEvent.RpcResult -> { + onResult(event.id) + removeQueuedMessage(event.id) + resolvePending(event.id, event.result, null) + } + + is WsEvent.RpcError -> { + onError(event.id) + if (pendingCalls[event.id]?.suppressErrorEvent == true) { + // Opt-in suppression: the caller already handles the + // failure through the CompletableDeferred, so the event + // copy would only surface a duplicate UI banner for + // optional features (e.g. "subagent.list" on gateways + // without the method, issue #1089). Default request() + // behavior still emits the event for shared consumers. + resolvePending(event.id, null, event.error) + return + } + removeQueuedMessage(event.id) + resolvePending(event.id, null, event.error) + } + + else -> {} + } + emitEvent(event) + } +} diff --git a/app/src/main/java/com/m57/hermescontrol/data/ws/WsTransport.kt b/app/src/main/java/com/m57/hermescontrol/data/ws/WsTransport.kt new file mode 100644 index 00000000..b425ca4d --- /dev/null +++ b/app/src/main/java/com/m57/hermescontrol/data/ws/WsTransport.kt @@ -0,0 +1,912 @@ +package com.m57.hermescontrol.data.ws + +import android.util.Log +import androidx.annotation.VisibleForTesting +import com.m57.hermescontrol.BuildConfig +import com.m57.hermescontrol.data.local.AuthManager +import com.m57.hermescontrol.data.remote.CookieManager +import com.m57.hermescontrol.data.remote.DashboardSessionTokenRefresher +import com.m57.hermescontrol.data.remote.NetworkMonitor +import com.m57.hermescontrol.data.remote.OkHttpProvider +import com.m57.hermescontrol.data.remote.await +import com.m57.hermescontrol.data.session.ActiveSessionHolder +import com.m57.hermescontrol.data.ws.contract.PromptSubmitParams +import com.m57.hermescontrol.data.ws.contract.RpcMethods +import com.m57.hermescontrol.data.ws.contract.SessionCorrectionParams +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.utf8Size +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger + +/** + * Socket lifecycle, authentication, outbound queueing, and network transport management. + * + * Extracted from [HermesWsClient] (issue #1376). + */ +internal class WsTransport( + private val scope: CoroutineScope, + private val outboundLock: Any, + private val reconnectPolicy: ReconnectPolicy, + private val connectionHealth: ConnectionHealth, + private val replayTracker: ReplayTracker, + private val rpcChannel: RpcChannel, + private val onRawMessage: (String) -> Unit, +) { + companion object { + private const val TAG = "HermesWsClient" + private const val MAX_OUTBOUND_MESSAGE_BYTES = 16 * 1024 * 1024 + private const val OUTBOUND_DRAIN_TIMEOUT_MS = 60_000L + } + + private val connectionGeneration = AtomicInteger(0) + private val connected = AtomicBoolean(false) + private val intentionalClose = AtomicBoolean(false) + private val acceptQueuedMessages = AtomicBoolean(true) + private val appInForeground = AtomicBoolean(true) + private val externalActivityConnectionLease = AtomicBoolean(false) + private val backgroundConnectionLease = AtomicBoolean(false) + private val messageQueue = ConcurrentLinkedQueue() + private val queuedMessagesById = ConcurrentHashMap() + + @Volatile + private var webSocket: WebSocket? = null + + @Volatile + private var closingSocket: WebSocket? = null + + @Volatile + private var reconnectJob: Job? = null + + @Volatile + private var outboundDrainJob: Job? = null + + private val networkCollectorStarted = AtomicBoolean(false) + + private val _connectionStatus = MutableStateFlow(ConnectionStatus.DISCONNECTED) + val connectionStatus: StateFlow = _connectionStatus.asStateFlow() + + @Volatile + var pendingReply: Boolean = false + internal set + + private val pendingPromptSubmits = ConcurrentHashMap.newKeySet() + + val isConnected: Boolean get() = connected.get() + + fun setAppForeground(foreground: Boolean) { + appInForeground.set(foreground) + if (!foreground) { + disconnectIfIdleInBackground() + return + } + connectionHealth.probeLivenessOnWake() + } + + fun acquireExternalActivityConnectionLease() { + externalActivityConnectionLease.set(true) + } + + fun releaseExternalActivityConnectionLease() { + externalActivityConnectionLease.set(false) + disconnectIfIdleInBackground() + } + + fun acquireBackgroundConnectionLease() { + backgroundConnectionLease.set(true) + } + + fun releaseBackgroundConnectionLease() { + backgroundConnectionLease.set(false) + disconnectIfIdleInBackground() + } + + internal fun hasBackgroundConnectionLease(): Boolean = backgroundConnectionLease.get() + + fun disconnectIfIdleInBackground() { + synchronized(outboundLock) { + if (!appInForeground.get() && + !externalActivityConnectionLease.get() && + !backgroundConnectionLease.get() && + !pendingReply && + !rpcChannel.hasPendingCalls() && + messageQueue.isEmpty() + ) { + disconnect() + } + } + } + + fun connect() { + if (networkCollectorStarted.compareAndSet(false, true)) { + scope.launch { + NetworkMonitor.networkChanges.collect { networkAvailable -> + reconnectForNetworkChange(networkAvailable) + } + } + scope.launch { + NetworkMonitor.transportChanges.collect { + connectionHealth.probeLivenessOnTransportChange( + onFailureAction = { cancelSocket() }, + ) + } + } + } + var socketToCancel: WebSocket? = null + val shouldOpen = + synchronized(outboundLock) { + acceptQueuedMessages.set(true) + if (_connectionStatus.value == ConnectionStatus.AUTH_EXPIRED) { + connectionGeneration.incrementAndGet() + reconnectJob?.cancel() + reconnectJob = null + outboundDrainJob?.cancel() + outboundDrainJob = null + socketToCancel = webSocket + webSocket = null + closingSocket = null + connected.set(false) + connectionHealth.stop() + } + if (connected.get()) { + Log.d(TAG, "Already connected — skipping") + return@synchronized false + } + if (_connectionStatus.value == ConnectionStatus.CONNECTING || + _connectionStatus.value == ConnectionStatus.RECONNECTING + ) { + Log.d(TAG, "Connection already in flight (${_connectionStatus.value}) — skipping") + return@synchronized false + } + // An explicit connect follows successful authentication and is + // therefore allowed to leave the terminal AUTH_EXPIRED state. + intentionalClose.set(false) + _connectionStatus.value = ConnectionStatus.CONNECTING + reconnectPolicy.reset() + true + } + if (socketToCancel != null) { + rpcChannel.rejectAllPending() + socketToCancel.cancel() + } + if (shouldOpen) openSocket() + } + + fun reconnectForNetworkChange( + networkAvailable: Boolean = NetworkMonitor.isConnected.value, + openReplacement: () -> Unit = ::openSocket, + ) { + val socketToCancel: WebSocket? + val shouldOpen: Boolean + synchronized(outboundLock) { + if (intentionalClose.get() || _connectionStatus.value == ConnectionStatus.AUTH_EXPIRED) { + return + } + Log.d(TAG, "Default network changed — reconnecting WebSocket") + reconnectPolicy.reset() + reconnectJob?.cancel() + reconnectJob = null + outboundDrainJob?.cancel() + outboundDrainJob = null + connectionGeneration.incrementAndGet() + socketToCancel = webSocket + webSocket = null + closingSocket = null + connected.set(false) + connectionHealth.stop() + shouldOpen = networkAvailable && AuthManager.isAutoReconnect() + _connectionStatus.value = + when { + !networkAvailable -> ConnectionStatus.NO_NETWORK + shouldOpen -> ConnectionStatus.RECONNECTING + else -> ConnectionStatus.DISCONNECTED + } + rpcChannel.rejectAllPending() + } + socketToCancel?.cancel() + if (shouldOpen) openReplacement() + } + + suspend fun refreshWsTicketIfNeeded(generation: Int): Boolean { + val isGated = + try { + AuthManager.serverStore.getLatestState().wsAuthParam == "ticket" + } catch (_: IllegalStateException) { + // serverStore not initialized yet (transient early call); treat + // as loopback so a stale-token refresh still runs. Log so a real + // misconfiguration isn't silently swallowed. + Log.w(TAG, "serverStore uninitialized during WS handshake; assuming non-gated") + false + } + if (!isGated) { + // The loopback dashboard token is regenerated on every server + // restart. Refresh it before each WebSocket handshake so automatic + // reconnect does not get stuck in AUTH_EXPIRED with a stale token. + val token = + runCatching { + DashboardSessionTokenRefresher.refreshAsync() + }.getOrNull() + synchronized(outboundLock) { + if (isCurrentGeneration(generation) && token != null) { + runCatching { AuthManager.setToken(token) } + } + } + return true + } + + val ticketResult = requestWsTicket() + if (!ticketResult.ticket.isNullOrBlank()) { + synchronized(outboundLock) { + if (!isCurrentGeneration(generation)) return false + AuthManager.setToken(ticketResult.ticket) + } + if (BuildConfig.DEBUG) Log.d(TAG, "WS ticket refreshed") + return true + } + val status = + when { + ticketResult.exception -> ConnectionStatus.RECONNECTING + ticketResult.httpCode == 401 || ticketResult.httpCode == 403 -> ConnectionStatus.AUTH_EXPIRED + ticketResult.httpCode == 408 || ticketResult.httpCode == 429 -> ConnectionStatus.RECONNECTING + ticketResult.httpCode != null && ticketResult.httpCode >= 500 -> ConnectionStatus.RECONNECTING + else -> ConnectionStatus.DISCONNECTED + } + return handleWsTicketRefreshFailure(status, generation) + } + + suspend fun mintWsTicket(): String? { + val isGated = + try { + AuthManager.serverStore.getLatestState().wsAuthParam == "ticket" + } catch (_: IllegalStateException) { + false + } + if (!isGated) { + DashboardSessionTokenRefresher.refreshAsync() + return AuthManager.getToken() + } + return requestWsTicket().ticket + } + + private data class TicketRequestResult( + val ticket: String?, + val httpCode: Int?, + val exception: Boolean = false, + ) + + private suspend fun requestWsTicket(): TicketRequestResult { + try { + val client = OkHttpProvider.probe + val request = + Request + .Builder() + .url(AuthManager.endpointForBuild().resolve("api/auth/ws-ticket").toString()) + .post("{}".toRequestBody()) + .build() + + val (code, body) = + withContext(Dispatchers.IO) { + if (CookieManager.isInitialized()) { + CookieManager.useStore(CookieManager.cookieJar.currentServer()) + } + client.newCall(request).await().use { resp -> + resp.code to resp.body.string() + } + } + + if (code in 200..299) { + // Real JSON decode — the ticket is base64url and may contain + // characters a naive [^"]+ regex cannot extract (see + // testGatedMode_parsesEscapedTicketFromRealJsonShape). + val parsed = + runCatching { + OkHttpProvider.json.decodeFromString(body) + }.getOrNull() + val ticket = parsed?.ticket + if (!ticket.isNullOrBlank()) { + return TicketRequestResult(ticket, null) + } + Log.w(TAG, "WS ticket mint failed: unparseable ticket response") + } else { + Log.w(TAG, "WS ticket mint failed: HTTP $code") + return TicketRequestResult(null, code) + } + } catch (e: Exception) { + // Include the stack: the exception class alone (e.g. + // NetworkOnMainThreadException) can't show WHICH caller on what + // thread tripped the mint (main-scope kanban events connect vs + // the WS reconnect path). + Log.w(TAG, "WS ticket mint failed: ${e.javaClass.simpleName}", e) + return TicketRequestResult(null, null, exception = true) + } + return TicketRequestResult(null, null) + } + + private fun handleWsTicketRefreshFailure( + status: ConnectionStatus, + generation: Int, + ): Boolean { + synchronized(outboundLock) { + if (!isCurrentGeneration(generation)) return false + _connectionStatus.value = status + if (status == ConnectionStatus.RECONNECTING) scheduleReconnect() + } + return false + } + + fun disconnect(clearPendingMessages: Boolean = false) { + ActiveSessionHolder.clear() + synchronized(outboundLock) { + intentionalClose.set(true) + acceptQueuedMessages.set(!clearPendingMessages) + connectionGeneration.incrementAndGet() + reconnectJob?.cancel() + reconnectJob = null + outboundDrainJob?.cancel() + outboundDrainJob = null + connectionHealth.cancelForegroundProbe() + connectionHealth.stop() + webSocket?.close(1000, "Client closed") + webSocket = null + closingSocket = null + rpcChannel.clearCapabilityRequests() + if (clearPendingMessages) { + backgroundConnectionLease.set(false) + messageQueue.clear() + queuedMessagesById.clear() + pendingPromptSubmits.clear() + pendingReply = false + replayTracker.clear() + } + connected.set(false) + _connectionStatus.value = ConnectionStatus.DISCONNECTED + } + } + + fun cancelSocket() { + synchronized(outboundLock) { + if (connected.get()) webSocket?.cancel() + } + } + + fun sendDirectFrame(frameJson: String): Boolean = + synchronized(outboundLock) { + val ws = webSocket + if (ws == null || !connected.get()) false else ws.send(frameJson) + } + + fun send( + request: JsonRpcRequest, + onSent: ((String) -> Unit)? = null, + queueIfDisconnected: Boolean = true, + ): String { + val id = request.id + val method = request.method + val json = rpcChannel.encodeRequest(request) + if (BuildConfig.DEBUG) { + Log.d(TAG, formatSafeOutgoingFrameLog(id, method, request.params.keys, json.length, queued = false)) + } + var reconnect = false + synchronized(outboundLock) { + if (method == WsMethods.PROMPT_SUBMIT) { + pendingPromptSubmits.add(id) + pendingReply = true + } + val ws = webSocket + if (ws != null && connected.get()) { + // Never replay send(true): the server may have executed it even if its response is lost. + if (ws.send(json)) { + onSent?.invoke(id) + } else { + if (webSocket !== ws || !acceptQueuedMessages.get()) return@synchronized + if (queueIfDisconnected && isRetryableMessage(json)) { + onSent?.invoke(id) + Log.w(TAG, "WS rejected outgoing message — queuing for reconnect") + queueMessage(id, json) + recoverRejectedSocket(ws) + } else { + Log.w(TAG, "WS rejected oversized outgoing message — not retrying") + } + } + } else if (acceptQueuedMessages.get() && queueIfDisconnected) { + if (isRetryableMessage(json)) { + onSent?.invoke(id) + Log.d(TAG, "WS disconnected — queuing message") + queueMessage(id, json) + reconnect = true + } else { + Log.w(TAG, "WS disconnected with oversized outgoing message — not queueing") + } + } + } + if (reconnect) connect() + return id + } + + fun removeQueuedMessage(id: String) { + synchronized(outboundLock) { + queuedMessagesById.remove(id)?.let(messageQueue::remove) + } + } + + fun onRpcResult(id: String) { + synchronized(outboundLock) { + pendingPromptSubmits.remove(id) + } + } + + fun onRpcError(id: String) { + synchronized(outboundLock) { + if (pendingPromptSubmits.remove(id) && pendingPromptSubmits.isEmpty()) { + pendingReply = false + disconnectIfIdleInBackground() + } + } + } + + fun observeEvent(event: WsEvent) { + when (event) { + is WsEvent.MessageStart, + is WsEvent.MessageToken, + is WsEvent.ThinkingDelta, + is WsEvent.ReasoningDelta, + is WsEvent.ToolStart, + -> { + pendingReply = true + } + + is WsEvent.MessageComplete -> { + pendingReply = false + disconnectIfIdleInBackground() + } + + else -> {} + } + } + + private fun isRetryableMessage(json: String): Boolean { + if (json.length > MAX_OUTBOUND_MESSAGE_BYTES) return false + if (json.length <= MAX_OUTBOUND_MESSAGE_BYTES / 4) return true + return json.utf8Size() <= MAX_OUTBOUND_MESSAGE_BYTES.toLong() + } + + private fun queueMessage( + id: String, + json: String, + ) { + if (queuedMessagesById.putIfAbsent(id, json) == null) messageQueue.add(json) + } + + private fun markQueuedMessageSent(json: String) { + queuedMessagesById.entries.removeIf { it.value == json } + } + + private fun recoverRejectedSocket(ws: WebSocket) { + connected.set(false) + if (closingSocket === ws) return + _connectionStatus.value = ConnectionStatus.RECONNECTING + if (ws.queueSize() == 0L) { + ws.cancel() + scheduleReconnect() + return + } + if (intentionalClose.get()) return + outboundDrainJob?.cancel() + outboundDrainJob = + scope.launch { + delay(OUTBOUND_DRAIN_TIMEOUT_MS) + synchronized(outboundLock) { + if (!connected.get() && !intentionalClose.get() && webSocket === ws) { + ws.cancel() + outboundDrainJob = null + scheduleReconnect() + } + } + } + } + + fun sendMessage( + sessionId: String, + text: String, + onSent: ((String) -> Unit)? = null, + queued: Boolean = false, + ): String = + send( + rpcChannel.createRequest( + method = RpcMethods.PROMPT_SUBMIT.name, + params = + PromptSubmitParams( + sessionId = sessionId, + text = text, + queued = queued.takeIf { it }, + ).let { rpcChannel.encodeParams(RpcMethods.PROMPT_SUBMIT, it) }, + ), + onSent = onSent, + ) + + fun sendRedirect( + sessionId: String, + text: String, + onSent: ((String) -> Unit)? = null, + ): String = + send( + rpcChannel.createRequest( + method = RpcMethods.SESSION_REDIRECT.name, + params = + SessionCorrectionParams( + sessionId = sessionId, + text = text, + ).let { rpcChannel.encodeParams(RpcMethods.SESSION_REDIRECT, it) }, + ), + onSent = onSent, + ) + + internal fun openSocket() { + val generation = + synchronized(outboundLock) { + if (intentionalClose.get()) return + connectionGeneration.incrementAndGet() + } + scope.launch(Dispatchers.IO) { + if (!refreshWsTicketIfNeeded(generation)) { + Log.w(TAG, "Aborting openSocket: WS ticket refresh failed") + return@launch + } + if (!isCurrentGeneration(generation)) return@launch + val url = AuthManager.wsUrl() + val safeUrl = url.replace(Regex("token=[^&]+"), "token=REDACTED") + if (BuildConfig.DEBUG) Log.d(TAG, "Connecting to $safeUrl") + + val request = Request.Builder().url(url).build() + val newSocket = OkHttpProvider.websocket.newWebSocket(request, WsListenerImpl(generation)) + synchronized(outboundLock) { + if (connectionGeneration.get() == generation && !intentionalClose.get()) { + webSocket = newSocket + } else { + newSocket.cancel() + } + } + } + } + + private fun isCurrentGeneration(generation: Int): Boolean = + connectionGeneration.get() == generation && !intentionalClose.get() + + private fun scheduleReconnect() { + synchronized(outboundLock) { + if (intentionalClose.get() || + _connectionStatus.value == ConnectionStatus.AUTH_EXPIRED || + reconnectJob?.isActive == true + ) { + return + } + if (!AuthManager.isAutoReconnect()) { + if (BuildConfig.DEBUG) Log.d(TAG, "Auto-reconnect disabled") + _connectionStatus.value = ConnectionStatus.DISCONNECTED + return + } + if (!NetworkMonitor.isConnected.value) { + Log.d(TAG, "No network available — delaying reconnect scheduling") + _connectionStatus.value = ConnectionStatus.NO_NETWORK + return + } + val reconnectDelay = reconnectPolicy.nextDelayMs() + reconnectPolicy.advance() + if (BuildConfig.DEBUG) Log.d(TAG, "Reconnecting in ${reconnectDelay}ms …") + + reconnectJob = + scope.launch { + delay(reconnectDelay) + val owner = currentCoroutineContext()[Job] + val shouldOpen = + synchronized(outboundLock) { + if (reconnectJob !== owner) { + false + } else { + reconnectJob = null + !intentionalClose.get() && + !connected.get() && + _connectionStatus.value != ConnectionStatus.AUTH_EXPIRED + } + } + if (shouldOpen) openSocket() + } + } + } + + private fun isTerminalAuthClose( + code: Int, + reason: String, + ): Boolean = + code == 4401 || code == 4403 || + reason.contains("unauthorized", ignoreCase = true) || + reason.startsWith("auth:", ignoreCase = true) + + private inner class WsListenerImpl( + private val generation: Int, + ) : WebSocketListener() { + private fun isCurrent(): Boolean = isCurrentGeneration(generation) + + override fun onOpen( + webSocket: WebSocket, + response: Response, + ) { + synchronized(outboundLock) { + if (!isCurrent()) { + webSocket.close(1000, "Superseded") + return + } + this@WsTransport.webSocket = webSocket + closingSocket = null + outboundDrainJob?.cancel() + outboundDrainJob = null + Log.i(TAG, "WebSocket opened") + connected.set(true) + _connectionStatus.value = ConnectionStatus.CONNECTED + reconnectPolicy.reset() + connectionHealth.start() + + while (true) { + val msg = messageQueue.peek() ?: break + if (!isRetryableMessage(msg)) { + Log.w(TAG, "Dropping oversized queued message") + messageQueue.poll() + markQueuedMessageSent(msg) + continue + } + if (BuildConfig.DEBUG) Log.d(TAG, formatSafeQueuedFrameLog(msg)) + if (!webSocket.send(msg)) { + recoverRejectedSocket(webSocket) + break + } + messageQueue.poll() + markQueuedMessageSent(msg) + } + replayTracker.triggerReplay() + } + disconnectIfIdleInBackground() + } + + override fun onMessage( + webSocket: WebSocket, + text: String, + ) { + if (!isCurrent() || this@WsTransport.webSocket !== webSocket) return + if (BuildConfig.DEBUG) Log.d(TAG, formatSafeIncomingFrameLog(text)) + connectionHealth.onInboundFrame() + onRawMessage(text) + } + + override fun onClosing( + webSocket: WebSocket, + code: Int, + reason: String, + ) { + synchronized(outboundLock) { + if (!isCurrent()) { + webSocket.close(code, reason) + return + } + closingSocket = webSocket + // Do NOT log [reason] — it may carry server-side context. + Log.d(TAG, "WebSocket closing: $code") + if (isTerminalAuthClose(code, reason)) { + _connectionStatus.value = ConnectionStatus.AUTH_EXPIRED + } + webSocket.close(code, reason) + } + } + + override fun onClosed( + webSocket: WebSocket, + code: Int, + reason: String, + ) { + synchronized(outboundLock) { + if (!isCurrent()) return + connectionGeneration.incrementAndGet() + closingSocket = null + outboundDrainJob?.cancel() + outboundDrainJob = null + if (this@WsTransport.webSocket === webSocket) this@WsTransport.webSocket = null + // Do NOT log [reason] — it may carry server-side context. The + // reason is still inspected internally to detect auth failures. + Log.i(TAG, "WebSocket closed: $code") + connected.set(false) + ActiveSessionHolder.clear() + connectionHealth.stop() + rpcChannel.clearCapabilityRequests() + if (isTerminalAuthClose(code, reason)) { + _connectionStatus.value = ConnectionStatus.AUTH_EXPIRED + } else if (_connectionStatus.value != ConnectionStatus.AUTH_EXPIRED) { + _connectionStatus.value = ConnectionStatus.RECONNECTING + scheduleReconnect() + } + } + } + + override fun onFailure( + webSocket: WebSocket, + t: Throwable, + response: Response?, + ) { + synchronized(outboundLock) { + if (!isCurrent()) return + connectionGeneration.incrementAndGet() + closingSocket = null + outboundDrainJob?.cancel() + outboundDrainJob = null + if (this@WsTransport.webSocket === webSocket) this@WsTransport.webSocket = null + // Log the exception class only — [Throwable.message] can leak URLs + // or headers. The message is still inspected internally for auth + // detection. + Log.e(TAG, "WebSocket failure: ${t.javaClass.simpleName}", t) + connected.set(false) + ActiveSessionHolder.clear() + connectionHealth.stop() + rpcChannel.clearCapabilityRequests() + val code = response?.code ?: 0 + if (code == 401 || code == 4401 || code == 4403 || + t.message?.contains("401") == true || + t.message?.contains("4401") == true || + t.message?.contains("4403") == true || + t.message?.contains("unauthorized", ignoreCase = true) == true + ) { + _connectionStatus.value = ConnectionStatus.AUTH_EXPIRED + } else if (_connectionStatus.value != ConnectionStatus.AUTH_EXPIRED) { + _connectionStatus.value = ConnectionStatus.RECONNECTING + scheduleReconnect() + } + } + } + } + + // ── Test seams (issue #1376 extraction support) ────────────────────── + + @VisibleForTesting + internal val connectedForTest: AtomicBoolean + get() = connected + + @VisibleForTesting + internal val intentionalCloseForTest: AtomicBoolean + get() = intentionalClose + + @VisibleForTesting + internal val connectionGenerationForTest: AtomicInteger + get() = connectionGeneration + + @VisibleForTesting + internal val messageQueueForTest: ConcurrentLinkedQueue + get() = messageQueue + + @VisibleForTesting + internal val outboundLockForTest: Any + get() = outboundLock + + @VisibleForTesting + internal val acceptQueuedMessagesForTest: AtomicBoolean + get() = acceptQueuedMessages + + @VisibleForTesting + internal val connectionStatusForTest: MutableStateFlow + get() = _connectionStatus + + @VisibleForTesting + internal var webSocketForTest: WebSocket? + get() = webSocket + set(value) { + webSocket = value + } + + @VisibleForTesting + internal var currentBackoffForTest: Long + get() = synchronized(outboundLock) { reconnectPolicy.currentBackoff } + set(value) { + synchronized(outboundLock) { + reconnectPolicy.currentBackoff = value + } + } + + @VisibleForTesting + internal var reconnectJobForTest: Job? + get() = reconnectJob + set(value) { + reconnectJob = value + } + + @VisibleForTesting + internal var outboundDrainJobForTest: Job? + get() = outboundDrainJob + set(value) { + outboundDrainJob = value + } + + @VisibleForTesting + internal fun setReconnectBackoffForTest(initialMillis: Long) { + synchronized(outboundLock) { + reconnectPolicy.setBackoffForTest(initialMillis) + } + } + + @VisibleForTesting + internal fun scheduleReconnectForTest() { + scheduleReconnect() + } + + @VisibleForTesting + internal fun createListenerForTest(generation: Int): WebSocketListener = WsListenerImpl(generation) + + @VisibleForTesting + internal fun setConnectionStatusForTest(status: ConnectionStatus) { + _connectionStatus.value = status + } + + @VisibleForTesting + internal fun formatSafeOutgoingFrameLog( + id: String, + method: String, + paramsKeys: Collection?, + byteLength: Int, + queued: Boolean = false, + ): String { + val prefix = if (queued) "→ (queued)" else "→" + val keys = paramsKeys?.joinToString(",", prefix = "[", postfix = "]") ?: "[]" + return "$prefix id=$id method=$method paramsKeys=$keys bytes=$byteLength" + } + + @VisibleForTesting + internal fun formatSafeQueuedFrameLog(msg: String): String = + try { + val req = OkHttpProvider.json.decodeFromString(msg) + formatSafeOutgoingFrameLog(req.id, req.method, req.params.keys, msg.length, queued = true) + } catch (_: Throwable) { + "→ (queued) frame bytes=${msg.length}" + } + + @VisibleForTesting + internal fun formatSafeIncomingFrameLog(text: String): String = + try { + val element = OkHttpProvider.json.parseToJsonElement(text) + if (element is JsonObject) { + val id = (element["id"] as? JsonPrimitive)?.content + val method = (element["method"] as? JsonPrimitive)?.content + val hasResult = element.containsKey("result") + val resultKeys = + (element["result"] as? JsonObject)?.keys?.joinToString( + ",", + prefix = "[", + postfix = "]", + ) + val errorObj = element["error"] as? JsonObject + val errorCode = (errorObj?.get("code") as? JsonPrimitive)?.content + val sb = StringBuilder("← ") + if (id != null) sb.append("id=").append(id).append(" ") + if (method != null) sb.append("method=").append(method).append(" ") + if (hasResult) sb.append("result=").append(resultKeys ?: "present").append(" ") + if (errorCode != null) sb.append("errorCode=").append(errorCode).append(" ") + sb.append("bytes=").append(text.length).toString() + } else { + "← non-object frame bytes=${text.length}" + } + } catch (_: Throwable) { + "← frame bytes=${text.length}" + } +} diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/ConnectionHealthTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/ConnectionHealthTest.kt new file mode 100644 index 00000000..19481076 --- /dev/null +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/ConnectionHealthTest.kt @@ -0,0 +1,518 @@ +package com.m57.hermescontrol.data.ws + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Test +import java.util.concurrent.atomic.AtomicInteger + +@OptIn(ExperimentalCoroutinesApi::class) +class ConnectionHealthTest { + private class Fixture( + val scope: CoroutineScope, + var connected: Boolean = true, + var pingDelayMs: Long = 100L, + var pingShouldFail: Boolean = false, + var pendingReplyState: Boolean = false, + var initialTimeMs: Long = 100_000L, + ) { + var currentTimeMs: Long = initialTimeMs + val pingCalls = mutableListOf() + val cancelCalls = AtomicInteger(0) + val logs = mutableListOf() + val debugLogs = mutableListOf() + + val health: ConnectionHealth = + ConnectionHealth( + scope = scope, + isConnected = { connected }, + pingRequest = { timeoutMs -> + pingCalls.add(timeoutMs) + if (pingShouldFail) { + throw IllegalStateException("Timed out after ${timeoutMs}ms") + } + if (pingDelayMs > 0) { + delay(pingDelayMs) + currentTimeMs += pingDelayMs + } + }, + cancelSocket = { + cancelCalls.incrementAndGet() + }, + pendingReply = { pendingReplyState }, + nowMs = { currentTimeMs }, + log = { logs.add(it) }, + debugLog = { debugLogs.add(it) }, + ) + } + + @Test + fun pingClockElapsedAndFlowUpdates() = + runTest { + val fixture = + Fixture( + scope = this, + pingDelayMs = 120L, + initialTimeMs = 1_000L, + ) + try { + assertEquals(1_000L, fixture.currentTimeMs) + val latency = fixture.health.ping(timeoutMs = 5000L) + assertEquals(120L, latency) + assertEquals(120L, fixture.health.lastLatencyMs.value) + assertEquals(1_120L, fixture.health.lastPongTimestamp) + assertEquals(listOf(5000L), fixture.pingCalls) + } finally { + fixture.health.stop() + } + } + + @Test + fun pingFailureLeavesPreviousLatencyUntouched() = + runTest { + val fixture = + Fixture( + scope = this, + pingDelayMs = 50L, + initialTimeMs = 5_000L, + ) + try { + val firstLatency = fixture.health.ping(timeoutMs = 5000L) + assertEquals(50L, firstLatency) + assertEquals(50L, fixture.health.lastLatencyMs.value) + + fixture.pingShouldFail = true + try { + fixture.health.ping(timeoutMs = 2000L) + fail("Expected ping failure") + } catch (expected: IllegalStateException) { + // Expected + } + + // Previous latency state is preserved + assertEquals(50L, fixture.health.lastLatencyMs.value) + } finally { + fixture.health.stop() + } + } + + @Test + fun stopResetsLatencyToNull() = + runTest { + val fixture = + Fixture( + scope = this, + pingDelayMs = 40L, + ) + try { + fixture.health.ping() + assertEquals(40L, fixture.health.lastLatencyMs.value) + + fixture.health.stop() + assertNull(fixture.health.lastLatencyMs.value) + } finally { + fixture.health.stop() + } + } + + @Test + fun onInboundFrameUpdatesPongTimestamp() = + runTest { + val fixture = + Fixture( + scope = this, + initialTimeMs = 10_000L, + ) + try { + assertEquals(0L, fixture.health.lastPongTimestamp) + + fixture.health.onInboundFrame() + assertEquals(10_000L, fixture.health.lastPongTimestamp) + + fixture.currentTimeMs = 25_000L + fixture.health.onInboundFrame() + assertEquals(25_000L, fixture.health.lastPongTimestamp) + } finally { + fixture.health.stop() + } + } + + @Test + fun isHealthyStrictlyEnforcesStaleThresholdOf30000() = + runTest { + val fixture = + Fixture( + scope = this, + connected = true, + initialTimeMs = 50_000L, + ) + try { + fixture.health.onInboundFrame() + assertEquals(50_000L, fixture.health.lastPongTimestamp) + + // now - lastPong < 30000 is healthy + fixture.currentTimeMs = 50_000L + 29_999L + assertTrue(fixture.health.isHealthy) + + // now - lastPong == 30000 is NOT healthy (< 30000 required) + fixture.currentTimeMs = 50_000L + 30_000L + assertFalse(fixture.health.isHealthy) + + // now - lastPong > 30000 is NOT healthy + fixture.currentTimeMs = 50_000L + 30_001L + assertFalse(fixture.health.isHealthy) + } finally { + fixture.health.stop() + } + } + + @Test + fun isHealthyDisconnectedGuard() = + runTest { + val fixture = + Fixture( + scope = this, + connected = true, + initialTimeMs = 10_000L, + ) + try { + fixture.health.onInboundFrame() + assertTrue(fixture.health.isHealthy) + + fixture.connected = false + assertFalse(fixture.health.isHealthy) + } finally { + fixture.health.stop() + } + } + + @Test + fun forceHealthCheckForTestWatchdogStrictThresholdCancelsSocket() = + runTest { + val fixture = + Fixture( + scope = this, + connected = true, + initialTimeMs = 100_000L, + ) + try { + fixture.health.onInboundFrame() + + // Within stale threshold (30000ms), no cancellation + fixture.health.forceHealthCheckForTest(30_000L) + assertEquals(0, fixture.cancelCalls.get()) + + // Exceeding stale threshold (> 30000ms), cancels socket + fixture.health.forceHealthCheckForTest(30_001L) + assertEquals(1, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun periodicHeartbeatCancelsSocketWhenStaleThresholdExceeded() = + runTest { + val fixture = + Fixture( + scope = this, + connected = true, + pingShouldFail = true, + pingDelayMs = 0L, + initialTimeMs = 10_000L, + ) + try { + fixture.health.start() + runCurrent() + fixture.health.onInboundFrame() + + // Advance by heartbeat interval (15000ms), still fresh + advanceTimeBy(15_000L) + fixture.currentTimeMs += 15_000L + runCurrent() + assertEquals(0, fixture.cancelCalls.get()) + + // Advance another heartbeat interval to 30000ms from last inbound frame + advanceTimeBy(15_000L) + fixture.currentTimeMs += 15_000L + runCurrent() + assertEquals(0, fixture.cancelCalls.get()) + + // Now time elapsed is > 30000ms (e.g. 30001ms) + fixture.currentTimeMs += 1L + advanceTimeBy(15_000L) + fixture.currentTimeMs += 15_000L + runCurrent() + assertEquals(1, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun setLastLatencyForTestSetsFlowDirectly() = + runTest { + val fixture = + Fixture( + scope = this, + ) + try { + assertNull(fixture.health.lastLatencyMs.value) + fixture.health.setLastLatencyForTest(75L) + assertEquals(75L, fixture.health.lastLatencyMs.value) + fixture.health.setLastLatencyForTest(null) + assertNull(fixture.health.lastLatencyMs.value) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnTransportChangeSuccess() = + runTest { + val fixture = + Fixture( + scope = this, + pingDelayMs = 25L, + ) + try { + var customActionCalled = false + fixture.health.probeLivenessOnTransportChange( + timeoutMs = 5000L, + onFailureAction = { customActionCalled = true }, + ) + runCurrent() + + assertEquals(listOf(5000L), fixture.pingCalls) + assertFalse(customActionCalled) + assertEquals(0, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnTransportChangeFailureInvokesCustomAction() = + runTest { + val fixture = + Fixture( + scope = this, + pingShouldFail = true, + ) + try { + var customActionCalled = false + fixture.health.probeLivenessOnTransportChange( + timeoutMs = 4000L, + onFailureAction = { customActionCalled = true }, + ) + runCurrent() + + assertEquals(listOf(4000L), fixture.pingCalls) + assertTrue(customActionCalled) + assertEquals(0, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnTransportChangeFailureInvokesDefaultCancelSocket() = + runTest { + val fixture = + Fixture( + scope = this, + pingShouldFail = true, + ) + try { + fixture.health.probeLivenessOnTransportChange( + timeoutMs = 5000L, + ) + runCurrent() + + assertEquals(listOf(5000L), fixture.pingCalls) + assertEquals(1, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnWakeIdleCancelsImmediatelyOnFailure() = + runTest { + val fixture = + Fixture( + scope = this, + pingShouldFail = true, + pendingReplyState = false, + ) + try { + fixture.health.probeLivenessOnWake(deferredOnce = false) + runCurrent() + + assertEquals(listOf(5000L), fixture.pingCalls) + assertEquals(1, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnWakePendingReplyFirstFailureDefers3000MsAndRetriesOnce() = + runTest { + val fixture = + Fixture( + scope = this, + pingShouldFail = true, + pendingReplyState = true, + ) + try { + fixture.health.probeLivenessOnWake(deferredOnce = false) + runCurrent() + + // First probe fails, but because pendingReply=true, it does not cancel immediately. + // It schedules a deferred retry after 3000ms. + assertEquals(1, fixture.pingCalls.size) + assertEquals(0, fixture.cancelCalls.get()) + + // Advance before 3000ms + advanceTimeBy(2999L) + runCurrent() + assertEquals(1, fixture.pingCalls.size) + assertEquals(0, fixture.cancelCalls.get()) + + // Advance to 3000ms - second ping is executed + advanceTimeBy(1L) + runCurrent() + assertEquals(2, fixture.pingCalls.size) + // Second probe fails, now it cancels + assertEquals(1, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnWakePendingReplySecondProbeSucceedsLeavesSocketOpen() = + runTest { + val fixture = + Fixture( + scope = this, + pingShouldFail = true, + pendingReplyState = true, + pingDelayMs = 10L, + ) + try { + fixture.health.probeLivenessOnWake(deferredOnce = false) + runCurrent() + + // First attempt failed, pending deferred retry in 3000ms + assertEquals(1, fixture.pingCalls.size) + assertEquals(0, fixture.cancelCalls.get()) + + // Before retry fires, recovery occurs: pings now succeed + fixture.pingShouldFail = false + + advanceTimeBy(3000L) + runCurrent() + + assertEquals(2, fixture.pingCalls.size) + assertEquals(0, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun cancelForegroundProbeCancelsPendingProbe() = + runTest { + val fixture = + Fixture( + scope = this, + pingShouldFail = true, + pendingReplyState = true, + ) + try { + fixture.health.probeLivenessOnWake(deferredOnce = false) + runCurrent() + + assertEquals(1, fixture.pingCalls.size) + assertEquals(0, fixture.cancelCalls.get()) + + // Cancel the scheduled retry + fixture.health.cancelForegroundProbe() + + advanceTimeBy(5000L) + runCurrent() + + // No second probe should have been made, no cancellation + assertEquals(1, fixture.pingCalls.size) + assertEquals(0, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnTransportChangeSuccessLogsAtDebugLevelWithoutWarning() = + runTest { + val fixture = + Fixture( + scope = this, + pingDelayMs = 25L, + ) + try { + fixture.health.probeLivenessOnTransportChange(timeoutMs = 5000L) + runCurrent() + advanceTimeBy(25L) + runCurrent() + + assertEquals(listOf(5000L), fixture.pingCalls) + assertEquals( + listOf( + "Network transport changed — probing WebSocket liveness", + "WebSocket liveness probe succeeded on new transport", + ), + fixture.debugLogs, + ) + assertTrue(fixture.logs.isEmpty()) + assertEquals(0, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } + + @Test + fun probeLivenessOnTransportChangeFailureLogsDebugAndWarningAndCancelsSocket() = + runTest { + val fixture = + Fixture( + scope = this, + pingShouldFail = true, + ) + try { + fixture.health.probeLivenessOnTransportChange(timeoutMs = 5000L) + runCurrent() + + assertEquals(listOf(5000L), fixture.pingCalls) + assertEquals( + listOf("Network transport changed — probing WebSocket liveness"), + fixture.debugLogs, + ) + assertEquals( + listOf("Transport change liveness probe failed — cancelling socket"), + fixture.logs, + ) + assertEquals(1, fixture.cancelCalls.get()) + } finally { + fixture.health.stop() + } + } +} diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientCharacterizationTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientCharacterizationTest.kt new file mode 100644 index 00000000..77ec705d --- /dev/null +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientCharacterizationTest.kt @@ -0,0 +1,532 @@ +package com.m57.hermescontrol.data.ws + +import android.util.Log +import com.m57.hermescontrol.data.local.AuthManager +import com.m57.hermescontrol.data.remote.CookieManager +import com.m57.hermescontrol.data.remote.DashboardSessionTokenRefresher +import com.m57.hermescontrol.data.remote.OkHttpProvider +import com.m57.hermescontrol.data.remote.buildFakePersistentCookieJar +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkObject +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Before +import org.junit.Test +import java.util.Collections +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +/** + * Characterization test suite capturing existing baseline behaviors and edge cases + * of [HermesWsClient] before extracting components in issue #1376. + */ +class HermesWsClientCharacterizationTest { + private lateinit var mockWebServer: MockWebServer + private val serverSockets = Collections.synchronizedList(mutableListOf()) + private lateinit var serverClosed: CountDownLatch + + @Before + fun setUp() { + serverSockets.clear() + serverClosed = CountDownLatch(1) + mockkStatic(Log::class) + every { Log.isLoggable(any(), any()) } returns false + every { Log.d(any(), any()) } returns 0 + every { Log.i(any(), any()) } returns 0 + every { Log.w(any(), any()) } returns 0 + every { Log.w(any(), any(), any()) } returns 0 + every { Log.e(any(), any()) } returns 0 + every { Log.e(any(), any(), any()) } returns 0 + + mockWebServer = MockWebServer() + mockWebServer.start() + + mockkObject(AuthManager) + every { AuthManager.wsUrl() } returns mockWebServer.url("/").toString().replace("http://", "ws://") + every { AuthManager.isAutoReconnect() } returns false + every { AuthManager.getSessionCookie() } returns null + every { AuthManager.serverStore } returns + mockk().also { + every { it.getLatestState() } returns + com.m57.hermescontrol.data.config + .ServerStoreState() + } + + CookieManager.setJarForTest(buildFakePersistentCookieJar()) + + mockkObject(DashboardSessionTokenRefresher) + coEvery { DashboardSessionTokenRefresher.refreshAsync() } returns null + every { DashboardSessionTokenRefresher.refresh() } returns null + + HermesWsClient.connectedForTest.set(false) + HermesWsClient.connectionStatusForTest.value = ConnectionStatus.DISCONNECTED + HermesWsClient.messageQueueForTest.clear() + HermesWsClient.disconnect(clearPendingMessages = true) + HermesWsClient.releaseExternalActivityConnectionLease() + HermesWsClient.releaseBackgroundConnectionLease() + HermesWsClient.setAppForeground(true) + HermesWsClient.acceptQueuedMessagesForTest.set(true) + HermesWsClient.clearCredentialWarning() + HermesWsClient.clearSeqWatermarks() + HermesWsClient.setLastLatencyForTest(null) + } + + @After + fun tearDown() { + HermesWsClient.rejectAllPending() + HermesWsClient.releaseExternalActivityConnectionLease() + HermesWsClient.releaseBackgroundConnectionLease() + HermesWsClient.disconnect(clearPendingMessages = true) + runBlocking { + withTimeout(3000) { + HermesWsClient.connectionStatus.first { it == ConnectionStatus.DISCONNECTED } + } + } + try { + if (serverSockets.isNotEmpty()) { + assertTrue("Server endpoint must finish closing", serverClosed.await(3, TimeUnit.SECONDS)) + } + mockWebServer.shutdown() + } finally { + unmockkAll() + } + } + + private fun connectClientToServer(onServerOpen: ((WebSocket) -> Unit)? = null): WebSocket { + val serverOpenLatch = CountDownLatch(1) + var serverWs: WebSocket? = null + + mockWebServer.enqueue( + MockResponse().withWebSocketUpgrade( + object : WebSocketListener() { + override fun onClosing( + webSocket: WebSocket, + code: Int, + reason: String, + ) { + webSocket.close(code, reason) + } + + override fun onClosed( + webSocket: WebSocket, + code: Int, + reason: String, + ) { + serverClosed.countDown() + } + + override fun onFailure( + webSocket: WebSocket, + t: Throwable, + response: okhttp3.Response?, + ) { + serverClosed.countDown() + } + + override fun onOpen( + webSocket: WebSocket, + response: okhttp3.Response, + ) { + serverSockets.add(webSocket) + serverWs = webSocket + onServerOpen?.invoke(webSocket) + serverOpenLatch.countDown() + } + }, + ), + ) + + HermesWsClient.connect() + runBlocking { + withTimeout(5000) { + HermesWsClient.connectionStatus.first { it == ConnectionStatus.CONNECTED } + } + } + assertTrue("Server WebSocket should open", serverOpenLatch.await(5, TimeUnit.SECONDS)) + return requireNotNull(serverWs) + } + + @Test + fun credentialWarning_updatedByGatewayReadyAndSessionInfo_retainedOnBlank_clearedExplicitly() { + assertNull(HermesWsClient.credentialWarning.value) + + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectedForTest.set(true) + val dummySocket = + mockk(relaxed = true) { + every { send(any()) } returns true + } + HermesWsClient.webSocketForTest = dummySocket + val generation = HermesWsClient.connectionGenerationForTest.get() + val listener = HermesWsClient.createListenerForTest(generation) + + // 1. GatewayReady with non-blank credential_warning updates flow + val gatewayReadyFrame = + """{"jsonrpc":"2.0","method":"event","params":{"type":"gateway.ready","payload":{"credential_warning":"Warning: Auth key expired"}}}""" + listener.onMessage(dummySocket, gatewayReadyFrame) + + runBlocking { + withTimeout(3000) { + HermesWsClient.credentialWarning.first { it == "Warning: Auth key expired" } + } + } + assertEquals("Warning: Auth key expired", HermesWsClient.credentialWarning.value) + + // 2. Blank or absent credential_warning does NOT clear the existing value. + // Deliver follow-up gateway.ready (absent) after blank and check warning is retained, + // then use session.info barrier to prove processing pipeline is drained. + val blankFrame = + """{"jsonrpc":"2.0","method":"event","params":{"type":"gateway.ready","payload":{"credential_warning":""}}}""" + listener.onMessage(dummySocket, blankFrame) + assertEquals("Warning: Auth key expired", HermesWsClient.credentialWarning.value) + + val absentFrame = + """{"jsonrpc":"2.0","method":"event","params":{"type":"gateway.ready","payload":{"version":"1.0"}}}""" + listener.onMessage(dummySocket, absentFrame) + assertEquals("Warning: Auth key expired", HermesWsClient.credentialWarning.value) + + // Barrier: a distinct non-blank session.info proves all preceding frames were processed in order + val barrierFrame = + """{"jsonrpc":"2.0","method":"event","params":{"type":"session.info","session_id":"s-barrier","payload":{"credential_warning":"Barrier reached"}}}""" + listener.onMessage(dummySocket, barrierFrame) + runBlocking { + withTimeout(3000) { + HermesWsClient.credentialWarning.first { it == "Barrier reached" } + } + } + assertEquals("Barrier reached", HermesWsClient.credentialWarning.value) + + // 3. SessionInfo with non-blank updates warning + val sessionInfoFrame = + """{"jsonrpc":"2.0","method":"event","params":{"type":"session.info","session_id":"s1","payload":{"credential_warning":"Session warning"}}}""" + listener.onMessage(dummySocket, sessionInfoFrame) + + runBlocking { + withTimeout(3000) { + HermesWsClient.credentialWarning.first { it == "Session warning" } + } + } + assertEquals("Session warning", HermesWsClient.credentialWarning.value) + + // 4. Explicit clear resets to null + HermesWsClient.clearCredentialWarning() + assertNull(HermesWsClient.credentialWarning.value) + } + + @Test + fun disconnect_resetsLastLatencyMsToNull() { + HermesWsClient.setLastLatencyForTest(42L) + assertEquals(42L, HermesWsClient.lastLatencyMs.value) + + HermesWsClient.disconnect() + assertNull(HermesWsClient.lastLatencyMs.value) + } + + @Test + fun rpcTimeout_emitsHermesRpcExceptionCodeMinusOneExactMessage() { + val serverWs = connectClientToServer() + assertNotNull(serverWs) + + val shortTimeoutMs = 150L + val deferred = HermesWsClient.request("custom.test_method", emptyMap(), timeoutMs = shortTimeoutMs) + + try { + runBlocking { + withTimeout(3000) { + deferred.await() + } + } + fail("Expected deferred to fail with timeout exception") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals(-1, e.code) + assertEquals("Request timed out: custom.test_method", e.message) + } + } + + @Test + fun rejectAllPending_failsRequestsWithCustomException_cancelsTimeout_clearsPending() { + connectClientToServer() + + val deferred = HermesWsClient.request("custom.hold", emptyMap(), timeoutMs = 10_000L) + val pendingIds = HermesWsClient.pendingCallIdsForTest() + assertEquals(1, pendingIds.size) + val reqId = pendingIds.first() + + val timeoutJob = HermesWsClient.pendingCallTimeoutForTest(reqId) + assertNotNull("Timeout job should be scheduled", timeoutJob) + assertTrue("Timeout job should be active", timeoutJob?.isActive == true) + + val customException = HermesWsClient.HermesRpcException("Custom forced rejection", code = -99) + HermesWsClient.rejectAllPending(customException) + + assertTrue( + "Pending map should be empty after rejectAllPending", + HermesWsClient.pendingCallIdsForTest().isEmpty(), + ) + assertTrue("Timeout job should be cancelled", timeoutJob?.isCancelled == true) + assertTrue("Deferred should be completed exceptionally", deferred.isCompleted) + + try { + runBlocking { deferred.await() } + fail("Expected custom exception") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals(-99, e.code) + assertEquals("Custom forced rejection", e.message) + } + } + + @Test + fun reconnectForNetworkChange_rejectsPendingCallsImmediately() { + connectClientToServer() + + val deferred = HermesWsClient.request("custom.in_flight", emptyMap(), timeoutMs = 30_000L) + assertEquals(1, HermesWsClient.pendingCallIdsForTest().size) + + // Network change triggers immediate rejectAllPending + HermesWsClient.reconnectForNetworkChange(networkAvailable = false) + + assertTrue("Pending map should be empty after network change", HermesWsClient.pendingCallIdsForTest().isEmpty()) + assertTrue("Deferred must be completed", deferred.isCompleted) + + try { + runBlocking { deferred.await() } + fail("Expected pending call to be rejected on network change") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals("Connection lost — request cancelled", e.message) + } + } + + @Test + fun disconnectAndListenerClosed_doNotRejectAllPending_deferredRemainsPendingUntilTimeoutOrReject() { + val serverWs = connectClientToServer() + + val shortTimeoutMs = 300L + val deferred = HermesWsClient.request("custom.deferred_call", emptyMap(), timeoutMs = shortTimeoutMs) + val reqId = HermesWsClient.pendingCallIdsForTest().first() + + // Drive onClosed on the CURRENT active generation before disconnect. + // In onClosed: isCurrent() is true (intentionalClose is still false, generation matches). + // This exercises the active listener onClosed branch. + val activeGeneration = HermesWsClient.connectionGenerationForTest.get() + val currentListener = HermesWsClient.createListenerForTest(activeGeneration) + val activeClientSocket = HermesWsClient.webSocketForTest ?: serverWs + + currentListener.onClosed(activeClientSocket, 1000, "Normal closure") + activeClientSocket.cancel() + + // Deferred remains pending after active onClosed + assertFalse("Deferred should still remain pending after onClosed", deferred.isCompleted) + assertTrue("Request ID should still be in pendingCalls", HermesWsClient.pendingCallIdsForTest().contains(reqId)) + + // Calling disconnect() does NOT invoke rejectAllPending() either + HermesWsClient.disconnect() + + assertFalse("Deferred should NOT be completed immediately by disconnect() alone", deferred.isCompleted) + assertTrue("Request ID should still be in pendingCalls", HermesWsClient.pendingCallIdsForTest().contains(reqId)) + + // Eventually, the per-request timeout triggers and resolves it + try { + runBlocking { + withTimeout(2000) { + deferred.await() + } + } + fail("Deferred should fail with timeout") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals(-1, e.code) + assertEquals("Request timed out: custom.deferred_call", e.message) + } + } + + @Test + fun combinedExternalAndBackgroundLeases_releaseOneKeepsOther_releaseBothDisconnectsWhenBackgrounded() { + connectClientToServer() + assertTrue(HermesWsClient.isConnected) + + // Acquire both leases + HermesWsClient.acquireExternalActivityConnectionLease() + HermesWsClient.acquireBackgroundConnectionLease() + assertTrue(HermesWsClient.hasBackgroundConnectionLease()) + + // Move to background - should NOT disconnect because leases are active + HermesWsClient.setAppForeground(false) + assertTrue("Should stay connected while leases are active", HermesWsClient.isConnected) + + // Release external lease - background lease still holds connection open + HermesWsClient.releaseExternalActivityConnectionLease() + assertTrue("Should stay connected with background lease active", HermesWsClient.isConnected) + + // Release background lease - both leases released, app backgrounded, idle -> disconnects + HermesWsClient.releaseBackgroundConnectionLease() + assertFalse("Should disconnect when all leases released in background", HermesWsClient.isConnected) + } + + @Test + fun delayedReplayAndLiveRace_incomingSeqBufferedBeforeReplayResponseAndDeduped() { + val sessionId = "session-characterization-race" + val serverOpenLatch = CountDownLatch(1) + val replayReceivedLatch = CountDownLatch(1) + val allowReplayResponseLatch = CountDownLatch(1) + val capturedReplayId = AtomicReference() + var serverSocketRef: WebSocket? = null + + // Enqueue MockResponse with WebSocketListener on MockWebServer + mockWebServer.enqueue( + MockResponse().withWebSocketUpgrade( + object : WebSocketListener() { + override fun onClosing( + webSocket: WebSocket, + code: Int, + reason: String, + ) { + webSocket.close(code, reason) + } + + override fun onClosed( + webSocket: WebSocket, + code: Int, + reason: String, + ) { + serverClosed.countDown() + } + + override fun onFailure( + webSocket: WebSocket, + t: Throwable, + response: okhttp3.Response?, + ) { + serverClosed.countDown() + } + + override fun onOpen( + webSocket: WebSocket, + response: okhttp3.Response, + ) { + serverSockets.add(webSocket) + serverSocketRef = webSocket + serverOpenLatch.countDown() + } + + override fun onMessage( + webSocket: WebSocket, + text: String, + ) { + if (text.contains(WsMethods.SESSION_EVENTS_SINCE)) { + val request = OkHttpProvider.json.parseToJsonElement(text) as JsonObject + val id = (request["id"] as JsonPrimitive).content + capturedReplayId.set(id) + replayReceivedLatch.countDown() + + if (allowReplayResponseLatch.await(5, TimeUnit.SECONDS)) { + val rpcId = requireNotNull(capturedReplayId.get()) + val idJson = JsonPrimitive(rpcId).toString() + val replayResponse = + """{"jsonrpc":"2.0","id":$idJson,"result":{"epoch":"epoch-1","latest_seq":12,"events":[{"type":"message.complete","session_id":"$sessionId","seq":11,"payload":{"text":"Replayed message 11"}},{"type":"message.complete","session_id":"$sessionId","seq":12,"payload":{"text":"Replayed message 12"}}]}}""" + webSocket.send(replayResponse) + } + } + } + }, + ), + ) + + // Set watermark at seq 10 and epoch-1 prior to connect + HermesWsClient.setSeqWatermark(sessionId, 10) + HermesWsClient.setReplayEpochForTest("epoch-1") + + // Prepare subscription with onSubscription latch before stimulus + val collectedEvents = Collections.synchronizedList(mutableListOf()) + val subscriptionLatch = CountDownLatch(1) + val collectLatch = CountDownLatch(2) + val liveProcessedLatch = CountDownLatch(1) + + val collectorJob = + kotlinx.coroutines.CoroutineScope(kotlinx.coroutines.Dispatchers.IO).launch { + HermesWsClient.events + .onSubscription { + subscriptionLatch.countDown() + }.collect { event -> + if (event is WsEvent.SessionInfo && event.data?.get("barrier") == "live-processed") { + liveProcessedLatch.countDown() + } + if (event is WsEvent.MessageComplete && event.sessionId == sessionId) { + collectedEvents.add(event) + collectLatch.countDown() + } + } + } + + try { + assertTrue("Collector subscription should be established", subscriptionLatch.await(3, TimeUnit.SECONDS)) + + // Connect client - real onOpen automatically arms replay + HermesWsClient.connect() + runBlocking { + withTimeout(5000) { + HermesWsClient.connectionStatus.first { it == ConnectionStatus.CONNECTED } + } + } + assertTrue("Server WebSocket should open", serverOpenLatch.await(5, TimeUnit.SECONDS)) + val serverSocket = requireNotNull(serverSocketRef) + + // Wait until the server receives session.events.since request + assertTrue("Server should receive replay request", replayReceivedLatch.await(5, TimeUnit.SECONDS)) + assertTrue("Replay should be in flight", HermesWsClient.isReplayInFlight()) + + // 1. Live frame arrived while replay is in flight (seq 12) + val liveFrame = + """{"jsonrpc":"2.0","method":"event","params":{"type":"message.complete","session_id":"$sessionId","seq":12,"payload":{"text":"Live message 12"}}}""" + serverSocket.send(liveFrame) + + // A following unsequenced frame proves the live notification reached the client + // before the replay response is released; unchanged watermark alone is not a barrier. + serverSocket.send( + """{"method":"event","params":{"type":"session.info","session_id":"$sessionId","payload":{"barrier":"live-processed"}}}""", + ) + assertTrue( + "Live frame must be processed before replay completes", + liveProcessedLatch.await(3, TimeUnit.SECONDS), + ) + assertEquals(10, HermesWsClient.getSeqWatermarks()[sessionId]) + assertEquals("Live frame should be held in replayHold while replay is in flight", 0, collectedEvents.size) + + // 2. Unblock replay response from server + allowReplayResponseLatch.countDown() + + // Wait for replay to deliver replayed events and drain held live events + assertTrue("Should collect replayed and unheld events", collectLatch.await(5, TimeUnit.SECONDS)) + + // Watermark should advance to 12 + assertEquals(12, HermesWsClient.getSeqWatermarks()[sessionId]) + + // Deduplication: held live frame seq 12 deduped against replay seq 12 + assertEquals(2, collectedEvents.size) + assertEquals("Replayed message 11", (collectedEvents[0] as WsEvent.MessageComplete).text) + assertEquals("Replayed message 12", (collectedEvents[1] as WsEvent.MessageComplete).text) + } finally { + allowReplayResponseLatch.countDown() + collectorJob.cancel() + } + } +} diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientLoggingSecurityTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientLoggingSecurityTest.kt index 29bfaa84..e4fcd588 100644 --- a/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientLoggingSecurityTest.kt +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientLoggingSecurityTest.kt @@ -11,7 +11,6 @@ import io.mockk.mockkStatic import io.mockk.unmockkAll import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.onSubscription import kotlinx.coroutines.launch @@ -30,11 +29,8 @@ import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test import java.util.Collections -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicInteger class HermesWsClientLoggingSecurityTest { private val capturedDebugLogs = Collections.synchronizedList(mutableListOf()) @@ -87,17 +83,9 @@ class HermesWsClientLoggingSecurityTest { CookieManager.setJarForTest(buildFakePersistentCookieJar()) - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as AtomicBoolean).set(false) - - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as AtomicBoolean).set(false) - - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as AtomicInteger).set(1) + HermesWsClient.connectedForTest.set(false) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionGenerationForTest.set(1) HermesWsClient.rejectAllPending() HermesWsClient.disconnect(clearPendingMessages = true) @@ -113,51 +101,20 @@ class HermesWsClientLoggingSecurityTest { unmockkAll() } - private fun createWsListener(generation: Int? = null): WebSocketListener { - val targetGen = - generation ?: run { - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as AtomicInteger).get() - } - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - return constructor.newInstance(targetGen) as WebSocketListener - } + private fun createWsListener(generation: Int? = null): WebSocketListener = + HermesWsClient.createListenerForTest(generation ?: HermesWsClient.connectionGenerationForTest.get()) private fun attachMockSocket(): WebSocket { val socket = mockk(relaxed = true) every { socket.send(any()) } returns true - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, socket) - - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as AtomicBoolean).set(true) - - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as AtomicBoolean).set(false) + HermesWsClient.webSocketForTest = socket + HermesWsClient.connectedForTest.set(true) + HermesWsClient.intentionalCloseForTest.set(false) return socket } - @Suppress("UNCHECKED_CAST") - private fun getPendingCalls(): ConcurrentHashMap { - val field = HermesWsClient::class.java.getDeclaredField("pendingCalls") - field.isAccessible = true - return field.get(HermesWsClient) as ConcurrentHashMap - } - - private fun getTimeoutJob(pendingCall: Any): Job? { - val field = pendingCall.javaClass.getDeclaredField("timeoutJob") - field.isAccessible = true - return field.get(pendingCall) as? Job - } - // ── Formatter unit tests ────────────────────────────────────────────────── @Test @@ -255,8 +212,7 @@ class HermesWsClientLoggingSecurityTest { val listener = createWsListener() val deferred = HermesWsClient.request("connectors.connect", mapOf("connector" to "synthetic_provider")) - val pendingCalls = getPendingCalls() - val reqId = pendingCalls.keys.firstOrNull() + val reqId = HermesWsClient.pendingCallIdsForTest().firstOrNull() assertNotNull("Pending call ID must exist", reqId) val sensitiveAuthUrl = @@ -332,8 +288,7 @@ class HermesWsClientLoggingSecurityTest { assertTrue("Event collector must subscribe", subscribedLatch.await(3, TimeUnit.SECONDS)) val deferred = HermesWsClient.request("session.steer", mapOf("session_id" to "sess-ownership-4001")) - val pendingCalls = getPendingCalls() - val reqId = pendingCalls.keys.firstOrNull() + val reqId = HermesWsClient.pendingCallIdsForTest().firstOrNull() assertNotNull("Pending call ID must exist", reqId) val errorJson = @@ -434,15 +389,9 @@ class HermesWsClientLoggingSecurityTest { fun `cancelling pending deferred cleans pending requests and cancels timeout job`() { attachMockSocket() val deferred = HermesWsClient.request("test.cancellation.method", mapOf("synthetic" to "fixture")) - val pendingCalls = getPendingCalls() - val entry = - pendingCalls.entries.firstOrNull { - it.value.javaClass.name - .contains("PendingCall") - } - assertNotNull("Pending call must be registered in pendingCalls map", entry) - val reqId = entry!!.key - val timeoutJob = getTimeoutJob(entry.value) + val reqId = HermesWsClient.pendingCallIdsForTest().firstOrNull() + assertNotNull("Pending call must be registered in pendingCalls map", reqId) + val timeoutJob = HermesWsClient.pendingCallTimeoutForTest(reqId!!) assertNotNull("Timeout job must be initialized", timeoutJob) assertTrue("Timeout job must be active while request is pending", timeoutJob!!.isActive) @@ -451,7 +400,7 @@ class HermesWsClientLoggingSecurityTest { assertTrue("Deferred must be cancelled", deferred.isCancelled) assertFalse( "Pending call must be removed from pendingCalls map on cancellation", - pendingCalls.containsKey(reqId), + HermesWsClient.pendingCallIdsForTest().contains(reqId), ) assertTrue("Timeout job must be cancelled on deferred cancellation", timeoutJob.isCancelled) } diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTest.kt index 14d5800e..7b5c1331 100644 --- a/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTest.kt +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTest.kt @@ -91,27 +91,15 @@ class HermesWsClientTest { every { DashboardSessionTokenRefresher.refresh() } returns null // Reset state - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - - val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus") - statusField.isAccessible = true - @Suppress("UNCHECKED_CAST") - (statusField.get(HermesWsClient) as MutableStateFlow).value = ConnectionStatus.DISCONNECTED - - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - (queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue).clear() + HermesWsClient.connectedForTest.set(false) + HermesWsClient.connectionStatusForTest.value = ConnectionStatus.DISCONNECTED + HermesWsClient.messageQueueForTest.clear() HermesWsClient.disconnect(clearPendingMessages = true) // Ensure it starts clean HermesWsClient.releaseExternalActivityConnectionLease() HermesWsClient.releaseBackgroundConnectionLease() HermesWsClient.setAppForeground(true) - val acceptQueuedMessagesField = HermesWsClient::class.java.getDeclaredField("acceptQueuedMessages") - acceptQueuedMessagesField.isAccessible = true - (acceptQueuedMessagesField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.acceptQueuedMessagesForTest.set(true) } @After @@ -350,10 +338,8 @@ class HermesWsClientTest { val socket = mockk() every { socket.send(any()) } returns true every { socket.close(any(), any()) } returns true - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket").apply { isAccessible = true } - val connectedField = HermesWsClient::class.java.getDeclaredField("connected").apply { isAccessible = true } - socketField.set(HermesWsClient, socket) - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.webSocketForTest = socket + HermesWsClient.connectedForTest.set(true) val sent = HermesWsClient.respondToServerRequest( @@ -384,10 +370,8 @@ class HermesWsClientTest { val socket = mockk() every { socket.send(any()) } returns true every { socket.close(any(), any()) } returns true - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket").apply { isAccessible = true } - val connectedField = HermesWsClient::class.java.getDeclaredField("connected").apply { isAccessible = true } - socketField.set(HermesWsClient, socket) - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.webSocketForTest = socket + HermesWsClient.connectedForTest.set(true) assertTrue(HermesWsClient.respondToServerRequestError("srq-abc123", -32601, "unsupported")) verify { @@ -416,29 +400,14 @@ class HermesWsClientTest { val staleSocket = mockk(relaxed = true) every { staleSocket.send(any()) } returns false - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, staleSocket) - - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) - - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - - val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus") - statusField.isAccessible = true - @Suppress("UNCHECKED_CAST") - (statusField.get(HermesWsClient) as MutableStateFlow).value = ConnectionStatus.CONNECTED + HermesWsClient.webSocketForTest = staleSocket + HermesWsClient.connectedForTest.set(true) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionStatusForTest.value = ConnectionStatus.CONNECTED HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello")) - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest assertFalse(HermesWsClient.isConnected) assertEquals(ConnectionStatus.DISCONNECTED, HermesWsClient.connectionStatus.value) assertEquals(1, queue.size) @@ -446,8 +415,7 @@ class HermesWsClientTest { @Test fun testSendRegistersRequestIdWhileOutboundLockHeld() { - val lockField = HermesWsClient::class.java.getDeclaredField("outboundLock").apply { isAccessible = true } - val outboundLock = lockField.get(HermesWsClient) + val outboundLock = HermesWsClient.outboundLockForTest var callbackHeldLock = false HermesWsClient.send("test.method", onSent = { callbackHeldLock = Thread.holdsLock(outboundLock) }) @@ -458,25 +426,13 @@ class HermesWsClientTest { @Test fun testNetworkChangeInvalidatesOldSocketAndOpensReplacementImmediately() { every { AuthManager.isAutoReconnect() } returns true - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectedForTest.set(true) val socket = mockk(relaxed = true) every { socket.send(any()) } returns true - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, socket) - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - val generation = generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger - val oldGeneration = generation.get() - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val staleListener = constructor.newInstance(oldGeneration) as WebSocketListener + HermesWsClient.webSocketForTest = socket + val oldGeneration = HermesWsClient.connectionGenerationForTest.get() + val staleListener = HermesWsClient.createListenerForTest(oldGeneration) var replacementOpens = 0 val deferred = HermesWsClient.request(WsMethods.PROCESS_LIST, mapOf("session_id" to "s1")) @@ -486,10 +442,10 @@ class HermesWsClientTest { verify(exactly = 1) { socket.cancel() } assertEquals(1, replacementOpens) - assertEquals(oldGeneration + 1, generation.get()) + assertEquals(oldGeneration + 1, HermesWsClient.connectionGenerationForTest.get()) assertFalse(HermesWsClient.isConnected) assertEquals(ConnectionStatus.RECONNECTING, HermesWsClient.connectionStatus.value) - assertEquals(null, socketField.get(HermesWsClient)) + assertEquals(null, HermesWsClient.webSocketForTest) assertTrue(deferred.isCompleted) } @@ -497,18 +453,10 @@ class HermesWsClientTest { fun testNetworkLossPreservesAuthExpiredStatus() = runBlocking { every { AuthManager.isAutoReconnect() } returns true - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus") - statusField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val status = statusField.get(HermesWsClient) as MutableStateFlow - status.value = ConnectionStatus.AUTH_EXPIRED + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED val socket = mockk(relaxed = true) - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, socket) + HermesWsClient.webSocketForTest = socket HermesWsClient.reconnectForNetworkChange(false) HermesWsClient.reconnectForNetworkChange(true) @@ -520,22 +468,13 @@ class HermesWsClientTest { @Test fun testBreakBeforeMakeDoesNotReplayAcceptedRequest() { every { AuthManager.isAutoReconnect() } returns true - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectedForTest.set(true) val oldSocket = mockk(relaxed = true) every { oldSocket.send(any()) } returns true - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, oldSocket) + HermesWsClient.webSocketForTest = oldSocket HermesWsClient.send(WsMethods.SUBSCRIPTION_CHANGE, mapOf("cancel" to true)) - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest assertTrue(queue.isEmpty()) HermesWsClient.reconnectForNetworkChange(false) @@ -545,13 +484,7 @@ class HermesWsClientTest { HermesWsClient.reconnectForNetworkChange(true) { replacementOpens++ } val replacementSocket = mockk(relaxed = true) every { replacementSocket.send(any()) } returns true - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - val generation = generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val replacementListener = constructor.newInstance(generation.get()) as WebSocketListener + val replacementListener = HermesWsClient.createListenerForTest(HermesWsClient.connectionGenerationForTest.get()) replacementListener.onOpen(replacementSocket, mockk(relaxed = true)) assertEquals(1, replacementOpens) @@ -563,18 +496,10 @@ class HermesWsClientTest { @Test fun testNetworkChangePreservesAuthExpiredSocket() { every { AuthManager.isAutoReconnect() } returns true - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus") - statusField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val status = statusField.get(HermesWsClient) as MutableStateFlow - status.value = ConnectionStatus.AUTH_EXPIRED + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED val socket = mockk(relaxed = true) - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, socket) + HermesWsClient.webSocketForTest = socket HermesWsClient.reconnectForNetworkChange() @@ -588,20 +513,13 @@ class HermesWsClientTest { every { AuthManager.isAutoReconnect() } returns true every { NetworkMonitor.isConnected } returns MutableStateFlow(true) - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.currentBackoffForTest = 1_000L - val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff") - backoffField.isAccessible = true - backoffField.setLong(HermesWsClient, 1_000L) + HermesWsClient.scheduleReconnectForTest() + HermesWsClient.scheduleReconnectForTest() - val reconnectMethod = HermesWsClient::class.java.getDeclaredMethod("scheduleReconnect") - reconnectMethod.isAccessible = true - reconnectMethod.invoke(HermesWsClient) - reconnectMethod.invoke(HermesWsClient) - - assertEquals(2_000L, backoffField.getLong(HermesWsClient)) + assertEquals(2_000L, HermesWsClient.currentBackoffForTest) } @Test @@ -610,35 +528,21 @@ class HermesWsClientTest { every { AuthManager.isAutoReconnect() } returns true every { NetworkMonitor.isConnected } returns MutableStateFlow(true) - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectedForTest.set(false) + HermesWsClient.currentBackoffForTest = 0L - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - - val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff") - backoffField.isAccessible = true - backoffField.setLong(HermesWsClient, 0L) - - val reconnectJobField = HermesWsClient::class.java.getDeclaredField("reconnectJob") - reconnectJobField.isAccessible = true val replacementJob = mockk(relaxed = true) - val lockField = HermesWsClient::class.java.getDeclaredField("outboundLock") - lockField.isAccessible = true - val lock = lockField.get(HermesWsClient) - val reconnectMethod = HermesWsClient::class.java.getDeclaredMethod("scheduleReconnect") - reconnectMethod.isAccessible = true + val lock = HermesWsClient.outboundLockForTest synchronized(lock) { - reconnectMethod.invoke(HermesWsClient) - reconnectJobField.set(HermesWsClient, replacementJob) + HermesWsClient.scheduleReconnectForTest() + HermesWsClient.reconnectJobForTest = replacementJob } Thread.sleep(100) - assertSame(replacementJob, reconnectJobField.get(HermesWsClient)) + assertSame(replacementJob, HermesWsClient.reconnectJobForTest) } @Test @@ -646,23 +550,15 @@ class HermesWsClientTest { val staleSocket = mockk(relaxed = true) every { staleSocket.send(any()) } returns false - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, staleSocket) - - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.webSocketForTest = staleSocket + HermesWsClient.connectedForTest.set(true) HermesWsClient.send( WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "x".repeat(16 * 1024 * 1024 + 1)), ) - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest assertTrue(queue.isEmpty()) io.mockk.verify(exactly = 0) { staleSocket.cancel() } } @@ -674,10 +570,7 @@ class HermesWsClientTest { mapOf("session_id" to "s1", "text" to "x".repeat(16 * 1024 * 1024 + 1)), ) - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest assertTrue(queue.isEmpty()) } @@ -687,24 +580,14 @@ class HermesWsClientTest { every { pressuredSocket.send(any()) } returns false every { pressuredSocket.queueSize() } returns 1024L - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, pressuredSocket) - - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) - - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.webSocketForTest = pressuredSocket + HermesWsClient.connectedForTest.set(true) + HermesWsClient.intentionalCloseForTest.set(false) HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello")) io.mockk.verify(exactly = 0) { pressuredSocket.cancel() } - val drainJobField = HermesWsClient::class.java.getDeclaredField("outboundDrainJob") - drainJobField.isAccessible = true - assertNotNull(drainJobField.get(HermesWsClient)) + assertNotNull(HermesWsClient.outboundDrainJobForTest) } @Test @@ -717,35 +600,18 @@ class HermesWsClientTest { every { pressuredSocket.send(any()) } returns false every { pressuredSocket.queueSize() } returns 1024L - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, pressuredSocket) - - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) - - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1) - - val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff") - backoffField.isAccessible = true - backoffField.setLong(HermesWsClient, 1_000L) + HermesWsClient.webSocketForTest = pressuredSocket + HermesWsClient.connectedForTest.set(true) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionGenerationForTest.set(1) + HermesWsClient.currentBackoffForTest = 1_000L HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello")) - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val listener = constructor.newInstance(1) as WebSocketListener + val listener = HermesWsClient.createListenerForTest(1) listener.onFailure(pressuredSocket, IOException("connection lost"), null) - assertEquals(2_000L, backoffField.getLong(HermesWsClient)) + assertEquals(2_000L, HermesWsClient.currentBackoffForTest) } @Test @@ -754,38 +620,22 @@ class HermesWsClientTest { every { pressuredSocket.send(any()) } returnsMany listOf(true, false) every { pressuredSocket.queueSize() } returns 1024L - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest queue.add("{\"jsonrpc\":\"2.0\",\"id\":\"1\"}") queue.add("{\"jsonrpc\":\"2.0\",\"id\":\"2\"}") - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionGenerationForTest.set(1) + HermesWsClient.webSocketForTest = pressuredSocket - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1) - - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, pressuredSocket) - - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val listener = constructor.newInstance(1) as WebSocketListener + val listener = HermesWsClient.createListenerForTest(1) listener.onOpen(pressuredSocket, mockk(relaxed = true)) assertFalse(HermesWsClient.isConnected) assertEquals(1, queue.size) assertTrue(queue.peek()?.contains("\"2\"") == true) io.mockk.verify(exactly = 0) { pressuredSocket.cancel() } - val drainJobField = HermesWsClient::class.java.getDeclaredField("outboundDrainJob") - drainJobField.isAccessible = true - assertNotNull(drainJobField.get(HermesWsClient)) + assertNotNull(HermesWsClient.outboundDrainJobForTest) } @Test @@ -794,29 +644,15 @@ class HermesWsClientTest { val currentSocket = mockk(relaxed = true) every { currentSocket.send(any()) } returns true - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, currentSocket) + HermesWsClient.webSocketForTest = currentSocket + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionGenerationForTest.set(2) - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(2) - - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest queue.add("{\"jsonrpc\":\"2.0\",\"id\":\"1\"}") - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val staleListener = constructor.newInstance(1) as WebSocketListener - val currentListener = constructor.newInstance(2) as WebSocketListener + val staleListener = HermesWsClient.createListenerForTest(1) + val currentListener = HermesWsClient.createListenerForTest(2) val response = mockk(relaxed = true) staleListener.onOpen(staleSocket, response) @@ -835,40 +671,24 @@ class HermesWsClientTest { false } - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, staleSocket) - - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) - - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.webSocketForTest = staleSocket + HermesWsClient.connectedForTest.set(true) + HermesWsClient.intentionalCloseForTest.set(false) HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello")) - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest assertTrue(queue.isEmpty()) assertEquals(ConnectionStatus.DISCONNECTED, HermesWsClient.connectionStatus.value) } @Test fun testDisconnectPreservesQueuedMessagesUnlessExplicitlyCleared() { - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.intentionalCloseForTest.set(false) HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello")) - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest assertEquals(1, queue.size) HermesWsClient.disconnect() @@ -886,16 +706,11 @@ class HermesWsClientTest { @Test fun testRejectAllPendingRemovesQueuedAwaitedRpc() { - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.intentionalCloseForTest.set(false) val deferred = HermesWsClient.request(WsMethods.PROCESS_LIST, mapOf("session_id" to "s1")) - val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue") - queueField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue + val queue = HermesWsClient.messageQueueForTest assertEquals(1, queue.size) HermesWsClient.rejectAllPending() @@ -910,26 +725,12 @@ class HermesWsClientTest { every { socket.send(any()) } returns false every { socket.queueSize() } returns 0L - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, socket) + HermesWsClient.webSocketForTest = socket + HermesWsClient.connectedForTest.set(true) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionGenerationForTest.set(1) - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) - - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1) - - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val listener = constructor.newInstance(1) as WebSocketListener + val listener = HermesWsClient.createListenerForTest(1) listener.onClosing(socket, 4401, "expired") HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello")) @@ -941,21 +742,10 @@ class HermesWsClientTest { @Test fun testSocketFailureDoesNotOverwriteAuthExpired() { val socket = mockk(relaxed = true) - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1) - val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus") - statusField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val status = statusField.get(HermesWsClient) as MutableStateFlow - status.value = ConnectionStatus.AUTH_EXPIRED - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val listener = constructor.newInstance(1) as WebSocketListener + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionGenerationForTest.set(1) + HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED + val listener = HermesWsClient.createListenerForTest(1) listener.onFailure(socket, IOException("network changed"), null) @@ -965,18 +755,10 @@ class HermesWsClientTest { @Test fun testExplicitConnectRetriesAfterAuthExpired() = runBlocking { - val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus") - statusField.isAccessible = true - @Suppress("UNCHECKED_CAST") - val status = statusField.get(HermesWsClient) as MutableStateFlow - status.value = ConnectionStatus.AUTH_EXPIRED - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED + HermesWsClient.connectedForTest.set(true) val oldSocket = mockk(relaxed = true) - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, oldSocket) + HermesWsClient.webSocketForTest = oldSocket mockWebServer.enqueue(MockResponse().withWebSocketUpgrade(object : WebSocketListener() {})) HermesWsClient.connect() @@ -1008,17 +790,16 @@ class HermesWsClientTest { HermesWsClient.disconnect(clearPendingMessages = true) mockWebServer.enqueue(MockResponse().withWebSocketUpgrade(object : WebSocketListener() {})) - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - val generation = generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger - val disconnectedGeneration = generation.get() + val disconnectedGeneration = HermesWsClient.connectionGenerationForTest.get() val replacementConnect = thread { HermesWsClient.connect() } val generationDeadline = System.currentTimeMillis() + 5_000 - while (generation.get() == disconnectedGeneration && System.currentTimeMillis() < generationDeadline) { + while (HermesWsClient.connectionGenerationForTest.get() == disconnectedGeneration && + System.currentTimeMillis() < generationDeadline + ) { Thread.sleep(10) } - assertTrue(generation.get() > disconnectedGeneration) - val currentGeneration = generation.get() + assertTrue(HermesWsClient.connectionGenerationForTest.get() > disconnectedGeneration) + val currentGeneration = HermesWsClient.connectionGenerationForTest.get() releaseRefresh.countDown() staleConnect.join(5_000) @@ -1029,7 +810,7 @@ class HermesWsClientTest { assertFalse(staleConnect.isAlive) assertFalse(replacementConnect.isAlive) - assertEquals(currentGeneration, generation.get()) + assertEquals(currentGeneration, HermesWsClient.connectionGenerationForTest.get()) assertEquals(ConnectionStatus.CONNECTED, HermesWsClient.connectionStatus.value) verify(exactly = 0) { AuthManager.setToken("stale-token") } } @@ -1441,12 +1222,10 @@ class HermesWsClientTest { runBlocking { withTimeout(5000) { HermesWsClient.connectionStatus.first { it == ConnectionStatus.CONNECTED } } } // After connect, backoff should be back to initial - val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff") - backoffField.isAccessible = true assertEquals( "Backoff should reset to initial after successful connect", 1000L, - backoffField.getLong(HermesWsClient), + HermesWsClient.currentBackoffForTest, ) } @@ -1999,26 +1778,11 @@ class HermesWsClientTest { fun testStreamingDedupPreservesSessionFallbackAndNumericSequenceSemantics() { // Issue #1163: exercise the real listener, including the early duplicate return. val socket = mockk(relaxed = true) - val intentionalClose = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalClose.isAccessible = true - (intentionalClose.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) - val generation = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generation.isAccessible = true - val constructor = - Class - .forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - .declaredConstructors - .single() - constructor.isAccessible = true - val listener = - constructor.newInstance( - (generation.get(HermesWsClient) as AtomicInteger).get(), - ) as WebSocketListener + HermesWsClient.intentionalCloseForTest.set(false) + val listener = HermesWsClient.createListenerForTest(HermesWsClient.connectionGenerationForTest.get()) // Directly install the reflection-built listener so onMessage runs even // though this test never performed a real OkHttp connect. - val socketField = HermesWsClient::class.java.getDeclaredField("webSocket") - socketField.isAccessible = true - socketField.set(HermesWsClient, socket) + HermesWsClient.webSocketForTest = socket mockkObject(EventParser) val seqValues = listOf("6", "6.9", "4294967302", "\"6\"", "null", "true", "{}", "[]") @@ -2039,18 +1803,10 @@ class HermesWsClientTest { @Test fun testTerminalCloseCode4403SetsAuthExpired() { val socket = mockk(relaxed = true) - val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose") - intentionalCloseField.isAccessible = true - (intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.intentionalCloseForTest.set(false) + HermesWsClient.connectionGenerationForTest.set(1) - val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration") - generationField.isAccessible = true - (generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1) - - val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl") - val constructor = listenerClass.declaredConstructors.single() - constructor.isAccessible = true - val listener = constructor.newInstance(1) as WebSocketListener + val listener = HermesWsClient.createListenerForTest(1) listener.onClosing(socket, 4403, "forbidden origin") assertEquals(ConnectionStatus.AUTH_EXPIRED, HermesWsClient.connectionStatus.value) @@ -2182,9 +1938,7 @@ class HermesWsClientTest { @Test fun testProbeLivenessOnTransportChangeCancelsSocketWhenPingFails() { - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true) + HermesWsClient.connectedForTest.set(true) val failureLatch = CountDownLatch(1) HermesWsClient.probeLivenessOnTransportChange( @@ -2199,9 +1953,7 @@ class HermesWsClientTest { @Test fun testProbeLivenessOnTransportChangeSkipsWhenDisconnected() { - val connectedField = HermesWsClient::class.java.getDeclaredField("connected") - connectedField.isAccessible = true - (connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false) + HermesWsClient.connectedForTest.set(false) var failureInvoked = false HermesWsClient.probeLivenessOnTransportChange( diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTypedCallTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTypedCallTest.kt index 9c7e23df..b8c7d01b 100644 --- a/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTypedCallTest.kt +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/HermesWsClientTypedCallTest.kt @@ -43,7 +43,6 @@ import org.junit.Assert.assertTrue import org.junit.Assert.fail import org.junit.Before import org.junit.Test -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicReference @@ -122,13 +121,6 @@ class HermesWsClientTypedCallTest { } } - @Suppress("UNCHECKED_CAST") - private fun getPendingCalls(): ConcurrentHashMap { - val field = HermesWsClient::class.java.getDeclaredField("pendingCalls") - field.isAccessible = true - return field.get(HermesWsClient) as ConcurrentHashMap - } - @Test fun callPutsExactFrameOnTheWire() { val opened = CountDownLatch(1) @@ -516,19 +508,25 @@ class HermesWsClientTypedCallTest { assertTrue("Server should receive request", requestReceived.await(5, TimeUnit.SECONDS)) - val pending = getPendingCalls() - assertEquals("Should have exactly 1 pending call while awaiting", 1, pending.size) + assertEquals( + "Should have exactly 1 pending call while awaiting", + 1, + HermesWsClient.pendingCallIdsForTest().size, + ) callJob.cancel() // Wait briefly for cancellation cleanup to execute withTimeout(2000) { - while (pending.isNotEmpty()) { + while (HermesWsClient.pendingCallIdsForTest().isNotEmpty()) { kotlinx.coroutines.delay(20) } } - assertTrue("Pending calls should be empty after caller coroutine is cancelled", pending.isEmpty()) + assertTrue( + "Pending calls should be empty after caller coroutine is cancelled", + HermesWsClient.pendingCallIdsForTest().isEmpty(), + ) } } } diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/ReconnectPolicyTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/ReconnectPolicyTest.kt new file mode 100644 index 00000000..2b3936b3 --- /dev/null +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/ReconnectPolicyTest.kt @@ -0,0 +1,98 @@ +package com.m57.hermescontrol.data.ws + +import org.junit.Assert.assertEquals +import org.junit.Test + +class ReconnectPolicyTest { + @Test + fun defaultInitialBackoffIs1000() { + val policy = ReconnectPolicy() + assertEquals(1000L, policy.currentBackoff) + } + + @Test + fun nextDelayMsUsesInjectedRandomAndFloorsAt300() { + // With random = 0.0, calculated delay is 0, floored to 300 (or ceiling if ceiling < 300) + val minPolicy = ReconnectPolicy(random = { 0.0 }) + assertEquals(300L, minPolicy.nextDelayMs()) + + // With random = 0.5 and ceiling = 1000, 0.5 * 1000 = 500 + val midPolicy = ReconnectPolicy(random = { 0.5 }) + assertEquals(500L, midPolicy.nextDelayMs()) + + // With random nearly 1.0 (0.999), 0.999 * 1000 = 999 + val maxPolicy = ReconnectPolicy(random = { 0.999 }) + assertEquals(999L, maxPolicy.nextDelayMs()) + } + + @Test + fun nextDelayMsRespectsCeilingCapsBelowFloorAndZero() { + // Ceiling <= 0 results in 0 + val zeroPolicy = ReconnectPolicy(random = { 0.5 }) + zeroPolicy.currentBackoff = 0L + assertEquals(0L, zeroPolicy.nextDelayMs()) + + val negativePolicy = ReconnectPolicy(random = { 0.5 }) + negativePolicy.currentBackoff = -100L + assertEquals(0L, negativePolicy.nextDelayMs()) + + // When ceiling is 100 (which is < 300 floor), delay should be capped to ceiling (100) + val lowCeilingPolicy = ReconnectPolicy(random = { 0.0 }) + lowCeilingPolicy.currentBackoff = 100L + assertEquals(100L, lowCeilingPolicy.nextDelayMs()) + } + + @Test + fun advanceDoublesBackoffUpToCapOf30000() { + val policy = ReconnectPolicy() + assertEquals(1000L, policy.currentBackoff) + + policy.advance() + assertEquals(2000L, policy.currentBackoff) + + policy.advance() + assertEquals(4000L, policy.currentBackoff) + + policy.advance() + assertEquals(8000L, policy.currentBackoff) + + policy.advance() + assertEquals(16000L, policy.currentBackoff) + + // Doubles past 30000 should cap exactly at 30000 (not speculative 15000 or 32000) + policy.advance() + assertEquals(30000L, policy.currentBackoff) + + policy.advance() + assertEquals(30000L, policy.currentBackoff) + } + + @Test + fun resetRestoresInitialBackoffOf1000() { + val policy = ReconnectPolicy() + policy.advance() + policy.advance() + assertEquals(4000L, policy.currentBackoff) + + policy.reset() + assertEquals(1000L, policy.currentBackoff) + } + + @Test + fun setBackoffForTestOverridesCurrentAndInitialBackoffDeterministically() { + val policy = ReconnectPolicy(random = { 0.5 }) + policy.setBackoffForTest(1500L) + + assertEquals(1500L, policy.currentBackoff) + // With test override, delay should be deterministic override value + assertEquals(1500L, policy.nextDelayMs()) + + // advance() still uses deterministic override if set, or advances from set initial/current + policy.advance() + assertEquals(1500L, policy.nextDelayMs()) + + // reset() retains deterministic override behavior or resets to overridden initial + policy.reset() + assertEquals(1500L, policy.nextDelayMs()) + } +} diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/ReplayTrackerTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/ReplayTrackerTest.kt new file mode 100644 index 00000000..7d1d3c4c --- /dev/null +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/ReplayTrackerTest.kt @@ -0,0 +1,555 @@ +package com.m57.hermescontrol.data.ws + +import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceParams +import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.concurrent.atomic.AtomicInteger + +@OptIn(ExperimentalCoroutinesApi::class) +class ReplayTrackerTest { + private val json = Json { ignoreUnknownKeys = true } + + private fun jsonElement(raw: String): JsonObject = json.parseToJsonElement(raw) as JsonObject + + @Test + fun `initial state has empty watermarks and no replay in flight`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = {}, + ) + + assertTrue(tracker.getSeqWatermarks().isEmpty()) + assertFalse(tracker.isReplayInFlight()) + } + + @Test + fun `watermarks can be set read and cleared without cancelling jobs`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = {}, + ) + + tracker.setSeqWatermark("s1", 10) + tracker.setSeqWatermark("s2", 25) + + assertEquals(mapOf("s1" to 10, "s2" to 25), tracker.getSeqWatermarks()) + + tracker.clearSeqWatermarks() + + assertTrue(tracker.getSeqWatermarks().isEmpty()) + assertFalse(tracker.isReplayInFlight()) + } + + @Test + fun `triggerReplay is no-op when watermarks are empty`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + var fetchCalled = false + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { + fetchCalled = true + SessionEventsSinceResult() + }, + emitEvent = {}, + ) + + tracker.triggerReplay() + + assertFalse(tracker.isReplayInFlight()) + testScope.advanceUntilIdle() + assertFalse(fetchCalled) + } + + @Test + fun `triggerReplay arms hold synchronously and sets replay in flight before coroutine runs`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val deferred = CompletableDeferred() + val emitted = mutableListOf() + + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { deferred.await() }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setSeqWatermark("s1", 5) + + // Arms hold synchronously + tracker.triggerReplay() + + assertTrue(tracker.isReplayInFlight()) + + // Calling triggerReplay again while in flight is ignored + tracker.triggerReplay() + assertTrue(tracker.isReplayInFlight()) + + // A live event arriving now for s1 must be parked into hold without advancing watermark + val liveMsg = WsEvent.MessageToken(token = "live", sessionId = "s1") + tracker.acceptLiveEvent("s1", 8, liveMsg) + + // Watermark has NOT advanced to 8 because replay is in flight and event is held + assertEquals(5, tracker.getSeqWatermarks()["s1"]) + assertTrue(emitted.isEmpty()) + + // Complete replay + deferred.complete(SessionEventsSinceResult()) + testScope.advanceUntilIdle() + + // Replay completed, held event released and emitted, watermark advanced + assertFalse(tracker.isReplayInFlight()) + assertEquals(listOf(liveMsg), emitted) + assertEquals(8, tracker.getSeqWatermarks()["s1"]) + } + + @Test + fun `isDuplicate drops equal or older seq and accepts newer seq`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = {}, + ) + + tracker.setSeqWatermark("s1", 10) + + // Same seq: duplicate + assertTrue(tracker.isDuplicate("s1", 10)) + // Older seq: duplicate + assertTrue(tracker.isDuplicate("s1", 5)) + // Newer seq: not duplicate + assertFalse(tracker.isDuplicate("s1", 11)) + // Unknown session defaults to seq 0 + assertFalse(tracker.isDuplicate("s2", 1)) + assertTrue(tracker.isDuplicate("s2", 0)) + } + + @Test + fun `acceptLiveEvent without in-flight replay emits event and advances watermark`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val emitted = mutableListOf() + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setSeqWatermark("s1", 10) + val event = WsEvent.MessageToken(token = "tok", sessionId = "s1") + + // Duplicate seq: dropped + tracker.acceptLiveEvent("s1", 10, event) + assertTrue(emitted.isEmpty()) + assertEquals(10, tracker.getSeqWatermarks()["s1"]) + + // Newer seq: emitted and updates watermark + tracker.acceptLiveEvent("s1", 12, event) + assertEquals(listOf(event), emitted) + assertEquals(12, tracker.getSeqWatermarks()["s1"]) + } + + @Test + fun `replay events are emitted in provided order and advance watermarks`() = + runTest { + val emitted = mutableListOf() + val event1 = + jsonElement( + """{"type":"message.token","session_id":"s1","seq":6,"payload":{"text":"a"}}""", + ) + val event2 = + jsonElement( + """{"type":"message.token","session_id":"s1","seq":7,"payload":{"text":"b"}}""", + ) + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { + SessionEventsSinceResult( + events = listOf(event1, event2), + ) + }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setSeqWatermark("s1", 5) + tracker.fetchReplayForTest() + + assertEquals(2, emitted.size) + assertEquals("a", (emitted[0] as WsEvent.MessageToken).token) + assertEquals("b", (emitted[1] as WsEvent.MessageToken).token) + assertEquals(7, tracker.getSeqWatermarks()["s1"]) + assertFalse(tracker.isReplayInFlight()) + } + + @Test + fun `replay events with seq at or below watermark are dropped`() = + runTest { + val emitted = mutableListOf() + val oldEvent = + jsonElement( + """{"type":"message.token","session_id":"s1","seq":5,"payload":{"text":"old"}}""", + ) + val newEvent = + jsonElement( + """{"type":"message.token","session_id":"s1","seq":6,"payload":{"text":"new"}}""", + ) + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { + SessionEventsSinceResult( + events = listOf(oldEvent, newEvent), + ) + }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setSeqWatermark("s1", 5) + tracker.fetchReplayForTest() + + assertEquals(1, emitted.size) + assertEquals("new", (emitted[0] as WsEvent.MessageToken).token) + assertEquals(6, tracker.getSeqWatermarks()["s1"]) + } + + @Test + fun `held live duplicates covered by replay are dropped and fresh held follow`() = + runTest { + val deferred = CompletableDeferred() + val emitted = mutableListOf() + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { deferred.await() }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setSeqWatermark("s1", 5) + tracker.triggerReplay() + + // Live event 6 arrives while replay in flight + val live6 = WsEvent.MessageToken(token = "live6", sessionId = "s1") + val live7 = WsEvent.MessageToken(token = "live7", sessionId = "s1") + tracker.acceptLiveEvent("s1", 6, live6) + tracker.acceptLiveEvent("s1", 7, live7) + + // Replay also returns event 6 + val replay6 = + jsonElement( + """{"type":"message.token","session_id":"s1","seq":6,"payload":{"text":"replay6"}}""", + ) + deferred.complete( + SessionEventsSinceResult( + events = listOf(replay6), + ), + ) + + testScheduler.advanceUntilIdle() + + // Replay 6 is emitted first, advancing watermark to 6. + // Held live 6 (seq 6 <= 6) is dropped as duplicate. + // Held live 7 (seq 7 > 6) is emitted and advances watermark to 7. + assertEquals(2, emitted.size) + assertEquals("replay6", (emitted[0] as WsEvent.MessageToken).token) + assertEquals("live7", (emitted[1] as WsEvent.MessageToken).token) + assertEquals(7, tracker.getSeqWatermarks()["s1"]) + assertFalse(tracker.isReplayInFlight()) + } + + @Test + fun `epoch change clears all watermarks updates epoch and requests transcript resync`() = + runTest { + val emitted = mutableListOf() + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { + SessionEventsSinceResult( + epoch = "epoch-2", + latestSeq = 50, + ) + }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setReplayEpochForTest("epoch-1") + tracker.setSeqWatermark("s1", 10) + tracker.setSeqWatermark("s2", 20) + + tracker.fetchReplayForTest() + + // Watermarks wiped + assertTrue(tracker.getSeqWatermarks().isEmpty()) + assertEquals(1, emitted.size) + assertEquals(WsEvent.TranscriptResyncRequired("s1"), emitted[0]) + } + + @Test + fun `empty or null epoch in result does not trigger epoch change`() = + runTest { + val emitted = mutableListOf() + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { + SessionEventsSinceResult( + epoch = "", + events = emptyList(), + ) + }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setReplayEpochForTest("epoch-1") + tracker.setSeqWatermark("s1", 10) + + tracker.fetchReplayForTest() + + assertEquals(10, tracker.getSeqWatermarks()["s1"]) + assertTrue(emitted.isEmpty()) + } + + @Test + fun `truncated resync fast-forwards watermark to latestSeq when epoch is same`() = + runTest { + val emitted = mutableListOf() + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { + SessionEventsSinceResult( + truncated = true, + latestSeq = 100, + epoch = "epoch-1", + ) + }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setReplayEpochForTest("epoch-1") + tracker.setSeqWatermark("s1", 10) + + tracker.fetchReplayForTest() + + // Fast-forwards to latestSeq and emits TranscriptResyncRequired + assertEquals(100, tracker.getSeqWatermarks()["s1"]) + assertEquals(listOf(WsEvent.TranscriptResyncRequired("s1")), emitted) + } + + @Test + fun `replay failure triggers resync invokes onReplayFailure and releases held events`() = + runTest { + val deferred = CompletableDeferred() + val emitted = mutableListOf() + val failedSessions = mutableListOf>() + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { + deferred.await() + }, + emitEvent = { emitted.add(it) }, + onReplayFailure = { sid, ex -> failedSessions.add(sid to ex) }, + ) + + tracker.setSeqWatermark("s1", 10) + tracker.triggerReplay() + + // Park a live event while waiting + val liveEvent = WsEvent.MessageToken(token = "held", sessionId = "s1") + tracker.acceptLiveEvent("s1", 15, liveEvent) + + // Fail replay + val testException = IllegalStateException("network error") + deferred.completeExceptionally(testException) + + testScheduler.advanceUntilIdle() + + // Failure emitted TranscriptResyncRequired, reported to callback, and released held events + assertEquals(1, failedSessions.size) + assertEquals("s1", failedSessions[0].first) + assertTrue(failedSessions[0].second is IllegalStateException) + assertEquals("network error", failedSessions[0].second.message) + + assertEquals(2, emitted.size) + assertEquals(WsEvent.TranscriptResyncRequired("s1"), emitted[0]) + assertEquals(liveEvent, emitted[1]) + assertEquals(15, tracker.getSeqWatermarks()["s1"]) + assertFalse(tracker.isReplayInFlight()) + } + + @Test + fun `MessageComplete event uses resolveStoredSessionId injection`() = + runTest { + val emitted = mutableListOf() + val msgCompleteJson = + jsonElement( + """{"type":"message.complete","session_id":"wire-sid","payload":{"text":"done"}}""", + ) + + val tracker = + ReplayTracker( + scope = this, + fetchEventsSince = { + SessionEventsSinceResult( + events = listOf(msgCompleteJson), + ) + }, + emitEvent = { emitted.add(it) }, + resolveStoredSessionId = { wireSid -> "resolved-$wireSid" }, + ) + + tracker.setSeqWatermark("wire-sid", 5) + tracker.fetchReplayForTest() + + assertEquals(1, emitted.size) + val complete = emitted[0] as WsEvent.MessageComplete + assertEquals("done", complete.text) + assertEquals("wire-sid", complete.sessionId) + assertEquals("resolved-wire-sid", complete.storedSessionId) + } + + @Test + fun `updateEpoch clears watermarks when epoch changes`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = {}, + ) + + tracker.setReplayEpochForTest("epoch-1") + tracker.setSeqWatermark("s1", 10) + + // Same epoch does not clear + tracker.updateEpoch("epoch-1") + assertEquals(10, tracker.getSeqWatermarks()["s1"]) + + // Empty / null does not clear + tracker.updateEpoch(null) + tracker.updateEpoch("") + assertEquals(10, tracker.getSeqWatermarks()["s1"]) + + // Different epoch clears watermarks + tracker.updateEpoch("epoch-2") + assertTrue(tracker.getSeqWatermarks().isEmpty()) + } + + @Test + fun `clear cancels in-flight job and resets all state`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val deferred = CompletableDeferred() + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { deferred.await() }, + emitEvent = {}, + ) + + tracker.setSeqWatermark("s1", 10) + tracker.triggerReplay() + assertTrue(tracker.isReplayInFlight()) + + tracker.clear() + + assertFalse(tracker.isReplayInFlight()) + assertTrue(tracker.getSeqWatermarks().isEmpty()) + } + + @Test + fun `acceptLiveEvent with eventFactory drops duplicate seq without invoking factory or emitting`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val emitted = mutableListOf() + var factoryInvoked = false + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setSeqWatermark("s1", 10) + + // Duplicate seq <= watermark must not invoke factory nor emit + tracker.acceptLiveEvent(sessionId = "s1", seq = 10) { + factoryInvoked = true + WsEvent.MessageToken(token = "duplicate", sessionId = "s1") + } + + assertFalse(factoryInvoked) + assertTrue(emitted.isEmpty()) + assertEquals(10, tracker.getSeqWatermarks()["s1"]) + } + + @Test + fun `acceptLiveEvent with eventFactory invokes factory once and preserves single-check ordering`() { + val testDispatcher = StandardTestDispatcher() + val testScope = TestScope(testDispatcher) + val emitted = mutableListOf() + val invocationCount = AtomicInteger(0) + val tracker = + ReplayTracker( + scope = testScope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { emitted.add(it) }, + ) + + tracker.setSeqWatermark("s1", 10) + val expectedEvent = WsEvent.MessageToken(token = "fresh", sessionId = "s1") + + // Factory intentionally injects an interleaved watermark advance (e.g. simulating race or side effect). + // Under single-check ordering, the initial pre-parse check (seq 11 > 10) succeeds, calls factory exactly once, + // and proceeds to emit and advance watermark without silently dropping due to an unexpected second check. + tracker.acceptLiveEvent(sessionId = "s1", seq = 11) { + invocationCount.incrementAndGet() + tracker.setSeqWatermark("s1", 99) + expectedEvent + } + + assertEquals(1, invocationCount.get()) + assertEquals(listOf(expectedEvent), emitted) + assertEquals(11, tracker.getSeqWatermarks()["s1"]) + } +} diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/RpcChannelTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/RpcChannelTest.kt new file mode 100644 index 00000000..5e1304a8 --- /dev/null +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/RpcChannelTest.kt @@ -0,0 +1,790 @@ +package com.m57.hermescontrol.data.ws + +import com.m57.hermescontrol.data.ws.contract.RpcMethod +import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Test + +@OptIn(ExperimentalCoroutinesApi::class) +class RpcChannelTest { + private val testJson = + Json { + ignoreUnknownKeys = true + encodeDefaults = false + } + + @Serializable + private data class SampleRequestParams( + val message: String, + val count: Int? = null, + ) + + @Serializable + private data class SampleResult( + val status: String, + val total: Int, + ) + + private val sampleMethod = + RpcMethod("test.sample", SampleRequestParams.serializer(), SampleResult.serializer()) + + private class FakeGateway( + scope: CoroutineScope, + json: Json, + outboundLock: Any = Any(), + val onResultCallback: (String) -> Unit = {}, + val onErrorCallback: (String) -> Unit = {}, + resolveStoredSessionId: (String?) -> String? = { it }, + val logParseFailureCallback: (Exception) -> Unit = {}, + decorateParams: (String, Map) -> Map = { _, p -> p }, + ) { + val sentRequests = mutableListOf() + val sentFrames = mutableListOf() + val removedQueuedIds = mutableListOf() + val rejectedLogs = mutableListOf>() + var idleCheckCount = 0 + var sendFrameReturnValue = true + + var immediateResolveOnSend: ((id: String, onSent: ((String) -> Unit)?) -> Unit)? = null + + val channel: RpcChannel = + RpcChannel( + scope = scope, + json = json, + sendRequest = { method, params, onSent -> + val id = "req-${sentRequests.size + 1}" + sentRequests.add(SentRequestRecord(id, method, params)) + onSent?.invoke(id) + immediateResolveOnSend?.invoke(id, onSent) + id + }, + sendFrame = { frame -> + sentFrames.add(frame) + sendFrameReturnValue + }, + removeQueuedMessage = { id -> + removedQueuedIds.add(id) + }, + onIdleCheck = { + idleCheckCount++ + }, + logRejected = { method, id -> + rejectedLogs.add(method to id) + }, + outboundLock = outboundLock, + onResult = onResultCallback, + onError = onErrorCallback, + resolveStoredSessionId = resolveStoredSessionId, + logParseFailure = logParseFailureCallback, + decorateParams = decorateParams, + ) + + data class SentRequestRecord( + val id: String, + val method: String, + val params: Map, + ) + } + + @Test + fun nextIdProducesIncreasingNonBlankStrings() { + val gateway = FakeGateway(CoroutineScope(CompletableDeferred()), testJson) + val firstId = gateway.channel.nextId() + val secondId = gateway.channel.nextId() + + assertEquals("1", firstId) + assertTrue(firstId.isNotBlank()) + assertTrue(secondId.isNotBlank()) + val firstNum = firstId.toLong() + val secondNum = secondId.toLong() + assertTrue(secondNum > firstNum) + assertEquals(firstNum + 1L, secondNum) + } + + @Test + fun requestSuccessResolvesDeferredAndCleansUpInOrder() = + runTest { + val gateway = FakeGateway(this, testJson) + val deferred = gateway.channel.request("user.ping", mapOf("key" to "value")) + val id = gateway.sentRequests.first().id + + assertTrue(gateway.channel.hasPendingCalls()) + assertTrue(gateway.channel.pendingCallIdsForTest().contains(id)) + assertNotNull(gateway.channel.pendingCallTimeoutForTest(id)) + + val dummyResult = JsonPrimitive("pong") + gateway.channel.resolvePending(id, dummyResult, null) + + val resolved = deferred.await() + assertEquals(dummyResult, resolved) + assertFalse(gateway.channel.hasPendingCalls()) + assertEquals(listOf(id), gateway.removedQueuedIds) + assertEquals(1, gateway.idleCheckCount) + assertNull(gateway.channel.pendingCallTimeoutForTest(id)) + } + + @Test + fun requestErrorResolvesWithHermesRpcExceptionAndErrorData() = + runTest { + val gateway = FakeGateway(this, testJson) + val deferred = gateway.channel.request("user.fail") + val id = gateway.sentRequests.first().id + + val errorData = buildJsonObject { put("reason", "auth_failed") } + val rpcError = JsonRpcError(code = 401, message = "Unauthorized", data = errorData) + + gateway.channel.resolvePending(id, null, rpcError) + + try { + deferred.await() + fail("Expected HermesRpcException") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals("Unauthorized", e.message) + assertEquals(401, e.code) + assertEquals(errorData, e.data) + } + + assertFalse(gateway.channel.hasPendingCalls()) + assertEquals(listOf(id), gateway.removedQueuedIds) + assertEquals(1, gateway.idleCheckCount) + } + + @Test + fun requestTimeoutFailsWithExactMessageAndCodeMinusOne() = + runTest { + val gateway = FakeGateway(this, testJson) + val timeoutMs = 5_000L + val deferred = gateway.channel.request("agent.turn", timeoutMs = timeoutMs) + val id = gateway.sentRequests.first().id + + runCurrent() + advanceTimeBy(4_999L) + runCurrent() + assertTrue(gateway.channel.hasPendingCalls()) + assertFalse(deferred.isCompleted) + + advanceTimeBy(1L) + runCurrent() + + assertTrue(deferred.isCompleted) + assertFalse(gateway.channel.hasPendingCalls()) + assertEquals(listOf(id), gateway.removedQueuedIds) + assertEquals(1, gateway.idleCheckCount) + + try { + deferred.await() + fail("Expected HermesRpcException on timeout") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals("Request timed out: agent.turn", e.message) + assertEquals(-1, e.code) + assertNull(e.data) + } + } + + @Test + fun deferredCancellationCancelsTimerRemovesQueuedAndTriggersIdle() = + runTest { + val gateway = FakeGateway(this, testJson) + val deferred = gateway.channel.request("slow.method", timeoutMs = 60_000L) + val id = gateway.sentRequests.first().id + + val timeoutJob = gateway.channel.pendingCallTimeoutForTest(id) + assertNotNull(timeoutJob) + assertFalse(timeoutJob!!.isCancelled) + + deferred.cancel() + runCurrent() + + assertTrue(deferred.isCancelled) + assertFalse(gateway.channel.hasPendingCalls()) + assertTrue(timeoutJob.isCancelled) + assertEquals(listOf(id), gateway.removedQueuedIds) + assertEquals(1, gateway.idleCheckCount) + + // Advancing past original timeout should do nothing further + advanceTimeBy(70_000L) + runCurrent() + assertEquals(1, gateway.idleCheckCount) + } + + @Test + fun suppressionFlagReflectedInIsErrorEventSuppressed() = + runTest { + val gateway = FakeGateway(this, testJson) + gateway.channel.request("normal.call", suppressErrorEvent = false) + val normalId = gateway.sentRequests[0].id + + gateway.channel.request("silent.call", suppressErrorEvent = true) + val silentId = gateway.sentRequests[1].id + + assertFalse(gateway.channel.isErrorEventSuppressed(normalId)) + assertTrue(gateway.channel.isErrorEventSuppressed(silentId)) + assertFalse(gateway.channel.isErrorEventSuppressed("non-existent-id")) + } + + @Test + fun rejectAllPendingFailsAllCallsWithProvidedExceptionAndNoIdleCheck() = + runTest { + val gateway = FakeGateway(this, testJson) + val deferred1 = gateway.channel.request("method.one") + val deferred2 = gateway.channel.request("method.two") + val id1 = gateway.sentRequests[0].id + val id2 = gateway.sentRequests[1].id + + val customException = HermesWsClient.HermesRpcException("Socket terminated abruptly", code = 1006) + + gateway.channel.rejectAllPending(customException) + + assertFalse(gateway.channel.hasPendingCalls()) + // Verified contract: rejectAllPending performs NO onIdleCheck + assertEquals(0, gateway.idleCheckCount) + assertEquals(setOf(id1, id2), gateway.removedQueuedIds.toSet()) + assertEquals(setOf("method.one" to id1, "method.two" to id2), gateway.rejectedLogs.toSet()) + + try { + deferred1.await() + fail("deferred1 should have failed") + } catch (e: HermesWsClient.HermesRpcException) { + assertSame(customException, e) + } + + try { + deferred2.await() + fail("deferred2 should have failed") + } catch (e: HermesWsClient.HermesRpcException) { + assertSame(customException, e) + } + } + + @Test + fun rejectAllPendingDefaultExceptionMatchesExpectedMessage() = + runTest { + val gateway = FakeGateway(this, testJson) + val deferred = gateway.channel.request("some.call") + + gateway.channel.rejectAllPending() + + try { + deferred.await() + fail("Expected default exception") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals("Connection lost — request cancelled", e.message) + } + } + + @Test + fun immediateResolutionDuringOnSentHandledSafely() = + runTest { + val gateway = FakeGateway(this, testJson) + gateway.immediateResolveOnSend = { id, _ -> + gateway.channel.resolvePending(id, "instant-result", null) + } + + val deferred = gateway.channel.request("instant.call") + val id = gateway.sentRequests.first().id + + assertTrue(deferred.isCompleted) + assertEquals("instant-result", deferred.await()) + assertFalse(gateway.channel.hasPendingCalls()) + assertEquals(listOf(id), gateway.removedQueuedIds) + assertEquals(1, gateway.idleCheckCount) + } + + @Test + fun typedCallSerializesParamsOmittingDefaultsAndDecodesResult() = + runTest { + val gateway = FakeGateway(this, testJson) + + val job = + async { + gateway.channel.call( + sampleMethod, + SampleRequestParams(message = "hello", count = null), + ) + } + + runCurrent() + assertEquals(1, gateway.sentRequests.size) + val sent = gateway.sentRequests.first() + assertEquals("test.sample", sent.method) + // encodeDefaults = false means count (null) is omitted from params + val encodedParams = + gateway.channel.encodeParams( + sampleMethod, + SampleRequestParams(message = "hello", count = null), + ) + assertFalse(encodedParams.containsKey("count")) + assertTrue(encodedParams.containsKey("message")) + + // Resolve pending with raw json representation + val rawResult = + buildJsonObject { + put("status", "ok") + put("total", 42) + put("extra_ignored_key", "bonus") + } + gateway.channel.resolvePending(sent.id, rawResult, null) + + val result = job.await() + assertEquals("ok", result.status) + assertEquals(42, result.total) + } + + @Test + fun typedCallCancellationPropagatesAndCancelsDeferred() = + runTest { + val gateway = FakeGateway(this, testJson) + + val job = + launch { + gateway.channel.call( + sampleMethod, + SampleRequestParams(message = "cancel-me"), + ) + } + + runCurrent() + val id = gateway.sentRequests.first().id + assertTrue(gateway.channel.hasPendingCalls()) + + job.cancel() + runCurrent() + + assertFalse(gateway.channel.hasPendingCalls()) + assertEquals(listOf(id), gateway.removedQueuedIds) + assertEquals(1, gateway.idleCheckCount) + } + + @Test + fun rawJsonNumericFidelityPreservedIntoResolve() = + runTest { + val gateway = FakeGateway(this, testJson) + + val deferred = + gateway.channel.requestTyped( + sampleMethod, + SampleRequestParams(message = "precision"), + ) + val id = gateway.sentRequests.first().id + + // Map with integer-value double, e.g. 177.0 -> preserves int semantics in toJsonElement + val mapResult = mapOf("status" to "precise", "total" to 177.0) + gateway.channel.resolvePending(id, mapResult, null) + + val raw = deferred.await() + val jsonElement = raw.toJsonElement() as JsonObject + assertEquals(177, jsonElement["total"]?.jsonPrimitive?.int) + + // Direct decode with injected channel json to verify round-trip + val decoded = testJson.decodeFromJsonElement(sampleMethod.result, jsonElement) + assertEquals(177, decoded.total) + assertEquals("precise", decoded.status) + } + + @Test + fun respondToServerRequestFormatsExactJsonRpcResponseAndPropagatesSink() { + val gateway = FakeGateway(CoroutineScope(CompletableDeferred()), testJson) + val serverReqId = "srv-999" + val payload = buildJsonObject { put("pong", true) } + + gateway.sendFrameReturnValue = true + val result = gateway.channel.respondToServerRequest(serverReqId, payload) + assertTrue(result) + assertEquals(1, gateway.sentFrames.size) + + val frame = testJson.decodeFromString(gateway.sentFrames.first()) + assertEquals("2.0", frame["jsonrpc"]?.jsonPrimitive?.content) + assertEquals(serverReqId, frame["id"]?.jsonPrimitive?.content) + assertEquals(payload, frame["result"]) + assertFalse(frame.containsKey("method")) + assertFalse(frame.containsKey("params")) + + // Blank id returns false immediately without sending frame + val blankResult = gateway.channel.respondToServerRequest(" ", payload) + assertFalse(blankResult) + assertEquals(1, gateway.sentFrames.size) + + // Sink returning false is propagated + gateway.sendFrameReturnValue = false + val falseResult = gateway.channel.respondToServerRequest("srv-1000", payload) + assertFalse(falseResult) + assertEquals(2, gateway.sentFrames.size) + } + + @Test + fun respondToServerRequestErrorFormatsExactJsonRpcErrorAndPropagatesSink() { + val gateway = FakeGateway(CoroutineScope(CompletableDeferred()), testJson) + val serverReqId = "srv-888" + + gateway.sendFrameReturnValue = true + val result = gateway.channel.respondToServerRequestError(serverReqId, -32601, "Method not found") + assertTrue(result) + assertEquals(1, gateway.sentFrames.size) + + val frame = testJson.decodeFromString(gateway.sentFrames.first()) + assertEquals("2.0", frame["jsonrpc"]?.jsonPrimitive?.content) + assertEquals(serverReqId, frame["id"]?.jsonPrimitive?.content) + assertFalse(frame.containsKey("result")) + val errorObj = frame["error"] as JsonObject + assertEquals(-32601, errorObj["code"]?.jsonPrimitive?.int) + assertEquals("Method not found", errorObj["message"]?.jsonPrimitive?.content) + + // Blank id returns false without sending frame + assertFalse(gateway.channel.respondToServerRequestError("", -1, "Bad")) + assertEquals(1, gateway.sentFrames.size) + + // Sink returning false is propagated + gateway.sendFrameReturnValue = false + val falseResult = gateway.channel.respondToServerRequestError("srv-889", -32000, "Error") + assertFalse(falseResult) + assertEquals(2, gateway.sentFrames.size) + } + + @Test + fun createRequestDecoratesOnceAllocatesIdAndEncodesOmittingDefaults() { + var decorateCount = 0 + val gateway = + FakeGateway( + scope = CoroutineScope(CompletableDeferred()), + json = testJson, + decorateParams = { method, params -> + decorateCount++ + params + ("decoratedMethod" to method) + }, + ) + + val request = gateway.channel.createRequest("test.action", mapOf("key" to "value")) + assertEquals("1", request.id) + assertEquals("test.action", request.method) + assertEquals(1, decorateCount) + assertEquals(JsonPrimitive("value"), request.params["key"]) + assertEquals(JsonPrimitive("test.action"), request.params["decoratedMethod"]) + + val encoded = gateway.channel.encodeRequest(request) + // Injected json with encodeDefaults = false omits default jsonrpc ("2.0") + assertFalse(encoded.contains("\"jsonrpc\"")) + assertTrue(encoded.contains("\"id\":\"1\"") || encoded.contains("\"id\": \"1\"")) + assertTrue(encoded.contains("\"method\":\"test.action\"") || encoded.contains("\"method\": \"test.action\"")) + + // An empty params map is omitted by encodeRequest under encodeDefaults = false + val plainGateway = FakeGateway(CoroutineScope(CompletableDeferred()), testJson) + val emptyParamsReq = plainGateway.channel.createRequest("test.empty", emptyMap()) + val encodedEmpty = plainGateway.channel.encodeRequest(emptyParamsReq) + assertFalse(encodedEmpty.contains("\"params\"")) + } + + @Test + fun handleIncomingFrameOrderRawJsonResolvedBeforeEventEmittedAndDoubleOnResult() = + runTest { + val onResultCalls = mutableListOf() + val events = mutableListOf() + val gateway = + FakeGateway( + scope = this, + json = testJson, + onResultCallback = { onResultCalls.add(it) }, + ) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { events.add(it) }, + ) + + val deferred = gateway.channel.request("client.calc") + val id = gateway.sentRequests.first().id + + var rawResolvedAtEventTime: Boolean? = null + var onResultCallsAtEventTime = 0 + val emitSink: (WsEvent) -> Unit = { event -> + events.add(event) + rawResolvedAtEventTime = deferred.isCompleted + onResultCallsAtEventTime = onResultCalls.size + } + + val incomingFrame = """{"jsonrpc":"2.0","id":"$id","result":{"ans":42.0}}""" + gateway.channel.handleIncomingFrame(incomingFrame, replayTracker, emitSink) + + // onResult is invoked twice baseline: once before raw resolve, and again before parsed RpcResult resolution + assertEquals(2, onResultCalls.size) + assertEquals(listOf(id, id), onResultCalls) + assertEquals(2, onResultCallsAtEventTime) + + // Raw deferred was settled BEFORE emitEvent was called + assertEquals(true, rawResolvedAtEventTime) + assertTrue(deferred.isCompleted) + + // Numeric fidelity preserved on result + val rawResult = deferred.await() + assertTrue(rawResult is JsonObject) + val rawObj = rawResult as JsonObject + assertEquals(JsonPrimitive(42.0), rawObj["ans"]) + + // Event emitted was RpcResult + assertEquals(1, events.size) + val rpcResult = events.first() as WsEvent.RpcResult + assertEquals(id, rpcResult.id) + } + + @Test + fun handleIncomingFrameRpcErrorInvokesCallbackAndRespectsSuppression() = + runTest { + val onErrorCalls = mutableListOf() + val events = mutableListOf() + val gateway = + FakeGateway( + scope = this, + json = testJson, + onErrorCallback = { onErrorCalls.add(it) }, + ) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { events.add(it) }, + ) + + // 1. Normal unsuppressed error + val normalDeferred = gateway.channel.request("call.normal", suppressErrorEvent = false) + val normalId = gateway.sentRequests[0].id + val errorFrame1 = + """{"jsonrpc":"2.0","id":"$normalId","error":{"code":-32603,"message":"Internal error"}}""" + gateway.channel.handleIncomingFrame(errorFrame1, replayTracker) { events.add(it) } + + assertEquals(listOf(normalId), onErrorCalls) + assertTrue(normalDeferred.isCompleted) + try { + normalDeferred.await() + fail("Expected HermesRpcException") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals(-32603, e.code) + assertEquals("Internal error", e.message) + } + assertEquals(1, events.size) + assertTrue(events.first() is WsEvent.RpcError) + assertEquals(normalId, (events.first() as WsEvent.RpcError).id) + + // 2. Suppressed error resolves deferred but emits NONE + val silentDeferred = gateway.channel.request("call.silent", suppressErrorEvent = true) + val silentId = gateway.sentRequests[1].id + val errorFrame2 = + """{"jsonrpc":"2.0","id":"$silentId","error":{"code":404,"message":"Not found"}}""" + gateway.channel.handleIncomingFrame(errorFrame2, replayTracker) { events.add(it) } + + assertEquals(listOf(normalId, silentId), onErrorCalls) + assertTrue(silentDeferred.isCompleted) + try { + silentDeferred.await() + fail("Expected HermesRpcException on silent deferred") + } catch (e: HermesWsClient.HermesRpcException) { + assertEquals(404, e.code) + } + // Event list size remains 1 (no RpcError event emitted for suppressed call) + assertEquals(1, events.size) + } + + @Test + fun handleIncomingFrameCapabilitySuppressionHidesEventUntilCleared() = + runTest { + val events = mutableListOf() + val gateway = FakeGateway(this, testJson) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { events.add(it) }, + ) + + val capId = "cap-42" + gateway.channel.registerCapabilityRequest(capId) + + val capResponseFrame = """{"jsonrpc":"2.0","id":"$capId","result":{"supported":true}}""" + gateway.channel.handleIncomingFrame(capResponseFrame, replayTracker) { events.add(it) } + + // Registered capability response is hidden from events + assertTrue(events.isEmpty()) + + // After clearing capabilities, subsequent responses with that ID are visible + gateway.channel.registerCapabilityRequest(capId) + gateway.channel.clearCapabilityRequests() + gateway.channel.handleIncomingFrame(capResponseFrame, replayTracker) { events.add(it) } + + assertEquals(1, events.size) + val rpcResult = events.first() as WsEvent.RpcResult + assertEquals(capId, rpcResult.id) + } + + @Test + fun handleIncomingFrameMalformedJsonInvokesParseFailureAndEmitsUnknown() = + runTest { + val parseFailures = mutableListOf() + val events = mutableListOf() + val gateway = + FakeGateway( + scope = this, + json = testJson, + logParseFailureCallback = { parseFailures.add(it) }, + ) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { events.add(it) }, + ) + + val malformed = "{ invalid json frame ::" + gateway.channel.handleIncomingFrame(malformed, replayTracker) { events.add(it) } + + assertEquals(1, parseFailures.size) + assertEquals(1, events.size) + val unknown = events.first() as WsEvent.Unknown + assertEquals(malformed, unknown.raw) + } + + @Test + fun handleIncomingFrameServerRequestClassifiedCorrectlyOverRpcResult() = + runTest { + val events = mutableListOf() + val gateway = FakeGateway(this, testJson) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { events.add(it) }, + ) + + // Normal srq frame with id + method should emit ServerRequest, not RpcResult + val srqFrame = """{"jsonrpc":"2.0","id":"srq-101","method":"clarify","params":{"text":"Confirm?"}}""" + gateway.channel.handleIncomingFrame(srqFrame, replayTracker) { events.add(it) } + + assertEquals(1, events.size) + val event = events.first() + assertTrue("Expected ServerRequest, got: $event", event is WsEvent.ServerRequest) + val srq = event as WsEvent.ServerRequest + assertEquals("srq-101", srq.id) + assertEquals("clarify", srq.method) + assertFalse(srq.replayed) + } + + @Test + fun handleIncomingFrameSequencedNotificationDeduplicatesViaReplayTracker() = + runTest { + val events = mutableListOf() + val gateway = FakeGateway(this, testJson) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { events.add(it) }, + ) + + val seqFrame = + """{"method":"events","params":{"type":"message.token","session_id":"s1","seq":11,"payload":{"text":"fresh"}}}""" + + // First delivery (fresh seq) emits once + gateway.channel.handleIncomingFrame(seqFrame, replayTracker) { events.add(it) } + assertEquals(1, events.size) + val tokenEvent = events.first() as WsEvent.MessageToken + assertEquals("fresh", tokenEvent.token) + assertEquals("s1", tokenEvent.sessionId) + + // Duplicate delivery (same seq 11) is dropped, emits none + gateway.channel.handleIncomingFrame(seqFrame, replayTracker) { events.add(it) } + assertEquals(1, events.size) + } + + @Test + fun handleIncomingFrameSettlesRawResultBeforeReplayingOpenRequests() = + runTest { + val order = mutableListOf() + val gateway = FakeGateway(this, testJson, onResultCallback = { order.add("result:$it") }) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = {}, + ) + val deferred = gateway.channel.request("session.resume") + val id = gateway.sentRequests.first().id + deferred.invokeOnCompletion { order.add("resolved") } + val frame = + """{"id":"$id","result":{"open_requests":[{"id":"srq-open","method":"clarify","params":{"text":"replayed"}}]}}""" + + gateway.channel.handleIncomingFrame(frame, replayTracker) { event -> + assertTrue("Raw deferred must settle before either event", deferred.isCompleted) + when (event) { + is WsEvent.ServerRequest -> { + assertTrue(event.replayed) + assertEquals("clarify", event.method) + assertEquals("replayed", event.params["text"]) + order.add("server:${event.id}") + } + + is WsEvent.RpcResult -> { + order.add("rpc:${event.id}") + } + + else -> { + fail("Unexpected event: $event") + } + } + } + + assertEquals(listOf("result:$id", "resolved", "server:srq-open", "result:$id", "rpc:$id"), order) + assertTrue(deferred.await() is JsonObject) + assertFalse(gateway.channel.hasPendingCalls()) + } + + @Test + fun handleIncomingFrameDecoratesStoredSessionIdOnMessageComplete() = + runTest { + val events = mutableListOf() + val gateway = + FakeGateway( + scope = this, + json = testJson, + resolveStoredSessionId = { sid -> "stored-$sid" }, + ) + val replayTracker = + ReplayTracker( + scope = this, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { events.add(it) }, + ) + + val completeFrame = + """{"method":"events","params":{"type":"message.complete","session_id":"active-session","payload":{"text":"done","reasoning":"reason"}}}""" + gateway.channel.handleIncomingFrame(completeFrame, replayTracker) { events.add(it) } + + assertEquals(1, events.size) + val completeEvent = events.first() as WsEvent.MessageComplete + assertEquals("done", completeEvent.text) + assertEquals("reason", completeEvent.reasoning) + assertEquals("active-session", completeEvent.sessionId) + assertEquals("stored-active-session", completeEvent.storedSessionId) + } +} diff --git a/app/src/test/java/com/m57/hermescontrol/data/ws/WsTransportTest.kt b/app/src/test/java/com/m57/hermescontrol/data/ws/WsTransportTest.kt new file mode 100644 index 00000000..fbb11bc4 --- /dev/null +++ b/app/src/test/java/com/m57/hermescontrol/data/ws/WsTransportTest.kt @@ -0,0 +1,489 @@ +package com.m57.hermescontrol.data.ws + +import android.util.Log +import com.m57.hermescontrol.data.local.AuthManager +import com.m57.hermescontrol.data.remote.NetworkMonitor +import com.m57.hermescontrol.data.session.ActiveSessionHolder +import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkObject +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import io.mockk.verify +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import okhttp3.Response +import okhttp3.WebSocket +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Before +import org.junit.Test +import java.util.concurrent.atomic.AtomicInteger + +@OptIn(ExperimentalCoroutinesApi::class) +class WsTransportTest { + private val testJson = + Json { + ignoreUnknownKeys = true + encodeDefaults = false + } + + private class Fixture( + val scope: CoroutineScope, + json: Json, + nowMs: () -> Long = { + if (scope is TestScope) { + scope.testScheduler.currentTime + } else { + System.nanoTime() / 1_000_000L + } + }, + ) { + val outboundLock = Any() + val rawMessages = mutableListOf() + val replayEmittedEvents = mutableListOf() + val reconnectPolicy = ReconnectPolicy() + + val rpcSentFrames = mutableListOf() + val rpcRemovedQueueIds = mutableListOf() + var rpcIdleChecks = 0 + + var transportRef: WsTransport? = null + private val rpcIdCounter = AtomicInteger(1) + + val connectionHealth = + ConnectionHealth( + scope = scope, + isConnected = { transportRef?.isConnected ?: false }, + pingRequest = { /* ping fake unit */ }, + cancelSocket = { transportRef?.cancelSocket() }, + pendingReply = { transportRef?.pendingReply ?: false }, + nowMs = nowMs, + ) + + val replayTracker = + ReplayTracker( + scope = scope, + fetchEventsSince = { SessionEventsSinceResult() }, + emitEvent = { replayEmittedEvents.add(it) }, + ) + + val rpcChannel = + RpcChannel( + scope = scope, + json = json, + sendRequest = { method, params, onSent -> + val id = "rpc-${rpcIdCounter.getAndIncrement()}" + val req = + rpcChannelRef?.createRequest(method, params)?.copy(id = id) + ?: JsonRpcRequest(id = id, method = method) + transportRef?.send(req, onSent) ?: id + }, + sendFrame = { frame -> + rpcSentFrames.add(frame) + transportRef?.sendDirectFrame(frame) ?: false + }, + removeQueuedMessage = { id -> + rpcRemovedQueueIds.add(id) + transportRef?.removeQueuedMessage(id) + }, + onIdleCheck = { + rpcIdleChecks++ + transportRef?.disconnectIfIdleInBackground() + }, + outboundLock = outboundLock, + onResult = { id -> transportRef?.onRpcResult(id) }, + onError = { id -> transportRef?.onRpcError(id) }, + ).also { rpcChannelRef = it } + + val transport = + WsTransport( + scope = scope, + outboundLock = outboundLock, + reconnectPolicy = reconnectPolicy, + connectionHealth = connectionHealth, + replayTracker = replayTracker, + rpcChannel = rpcChannel, + onRawMessage = { rawMessages.add(it) }, + ).also { transportRef = it } + + companion object { + private var rpcChannelRef: RpcChannel? = null + } + } + + @Before + fun setUp() { + mockkStatic(Log::class) + every { Log.d(any(), any()) } returns 0 + every { Log.i(any(), any()) } returns 0 + every { Log.w(any(), any()) } returns 0 + every { Log.w(any(), any(), any()) } returns 0 + every { Log.e(any(), any(), any()) } returns 0 + + ActiveSessionHolder.clear() + + mockkObject(AuthManager) + every { AuthManager.isAutoReconnect() } returns false + + mockkObject(NetworkMonitor) + every { NetworkMonitor.isConnected } returns MutableStateFlow(true) + } + + @After + fun tearDown() { + ActiveSessionHolder.clear() + unmockkAll() + } + + @Test + fun sendAcceptCallbackRunsUnderOutboundLockAfterSocketAccept() = + runTest { + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + every { socket.send(any()) } returns true + + fixture.transport.connectedForTest.set(true) + fixture.transport.webSocketForTest = socket + + var callbackRan = false + var callbackThreadHoldsLock = false + val request = JsonRpcRequest(id = "req-1", method = "session.list") + + val id = + fixture.transport.send( + request = request, + onSent = { + callbackRan = true + callbackThreadHoldsLock = Thread.holdsLock(fixture.outboundLock) + }, + ) + + assertEquals("req-1", id) + assertTrue(callbackRan) + assertTrue(callbackThreadHoldsLock) + verify(exactly = 1) { socket.send(any()) } + assertTrue(fixture.transport.messageQueueForTest.isEmpty()) + } + + @Test + fun rejectedRetryableSendEnqueuesAndCancelDoesNotAutoRedialWhenAutoReconnectDisabled() = + runTest { + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + every { socket.send(any()) } returns false + every { socket.queueSize() } returns 0L + + fixture.transport.connectedForTest.set(true) + fixture.transport.intentionalCloseForTest.set(false) + fixture.transport.connectionGenerationForTest.set(1) + fixture.transport.webSocketForTest = socket + + var onSentCalled = false + var callbackThreadHoldsLock = false + val request = JsonRpcRequest(id = "req-queued", method = "prompt.submit") + + fixture.transport.send( + request = request, + onSent = { + onSentCalled = true + callbackThreadHoldsLock = Thread.holdsLock(fixture.outboundLock) + }, + queueIfDisconnected = true, + ) + + // When socket rejects, request is pushed back into retryable queue + val queue = fixture.transport.messageQueueForTest + assertEquals(1, queue.size) + assertTrue(queue.peek()?.contains("req-queued") == true) + assertTrue(onSentCalled) + assertTrue(callbackThreadHoldsLock) + assertFalse(fixture.transport.isConnected) + assertEquals(ConnectionStatus.DISCONNECTED, fixture.transport.connectionStatus.value) + assertNull(fixture.transport.reconnectJobForTest) + verify(exactly = 1) { socket.cancel() } + } + + @Test + fun staleOnMessageIgnoredWhileCurrentGenerationFrameUpdatesHealthAndRawCallback() = + runTest { + val fixture = + Fixture( + scope = this, + json = testJson, + nowMs = { testScheduler.currentTime }, + ) + val currentSocket = mockk(relaxed = true) + val staleSocket = mockk(relaxed = true) + + fixture.transport.connectedForTest.set(true) + fixture.transport.connectionGenerationForTest.set(2) + fixture.transport.webSocketForTest = currentSocket + + val initialTimestamp = fixture.connectionHealth.lastPongTimestamp + assertEquals(0L, initialTimestamp) + + val staleListener = fixture.transport.createListenerForTest(1) + val currentListener = fixture.transport.createListenerForTest(2) + + advanceTimeBy(1_000L) + // Message from stale listener/generation should be discarded + staleListener.onMessage(staleSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"old\"}") + staleListener.onMessage(currentSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"old-mismatched-gen\"}") + assertTrue(fixture.rawMessages.isEmpty()) + assertEquals(0L, fixture.connectionHealth.lastPongTimestamp) + + advanceTimeBy(1_000L) + // Message from current generation but mismatched socket should be discarded + currentListener.onMessage(staleSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"wrong-socket\"}") + assertTrue(fixture.rawMessages.isEmpty()) + assertEquals(0L, fixture.connectionHealth.lastPongTimestamp) + + advanceTimeBy(1_000L) + val expectedTimestamp = testScheduler.currentTime + // Matching generation and matching active socket passes through and updates health + currentListener.onMessage(currentSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"valid\"}") + assertEquals(listOf("{\"jsonrpc\":\"2.0\",\"id\":\"valid\"}"), fixture.rawMessages) + assertEquals(expectedTimestamp, fixture.connectionHealth.lastPongTimestamp) + } + + @Test + fun staleOnOpenClosesSupersededWithoutChangingOwnershipOrFlushingQueue() = + runTest { + val fixture = Fixture(this, testJson) + val staleSocket = mockk(relaxed = true) + val currentSocket = mockk(relaxed = true) + val response = mockk(relaxed = true) + + fixture.transport.intentionalCloseForTest.set(false) + fixture.transport.connectionGenerationForTest.set(2) + fixture.transport.webSocketForTest = currentSocket + fixture.transport.messageQueueForTest.add("{\"jsonrpc\":\"2.0\",\"id\":\"queued-1\"}") + + val staleListener = fixture.transport.createListenerForTest(1) + staleListener.onOpen(staleSocket, response) + + // Stale onOpen must close the stale socket with 1000 ("Superseded") + verify(exactly = 1) { staleSocket.close(1000, "Superseded") } + // Must not change active socket or flush the queue on stale socket + verify(exactly = 0) { staleSocket.send(any()) } + assertEquals(currentSocket, fixture.transport.webSocketForTest) + assertEquals(1, fixture.transport.messageQueueForTest.size) + } + + @Test + fun authClosingCodeSetsTerminalAuthExpiredAndDoesNotAutoRedial() = + runTest { + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + every { socket.send(any()) } returns true + + fixture.transport.connectedForTest.set(true) + fixture.transport.intentionalCloseForTest.set(false) + fixture.transport.connectionGenerationForTest.set(1) + fixture.transport.webSocketForTest = socket + + // Register a pending RPC call to verify it is NOT newly rejected + val pendingDeferred = fixture.rpcChannel.request("custom.in_flight", emptyMap()) + assertFalse(pendingDeferred.isCompleted) + + val listener = fixture.transport.createListenerForTest(1) + listener.onClosing(socket, 4401, "unauthorized") + + assertEquals(ConnectionStatus.AUTH_EXPIRED, fixture.transport.connectionStatus.value) + assertTrue(fixture.transport.isConnected) + assertNull(fixture.transport.reconnectJobForTest) + assertFalse(pendingDeferred.isCompleted) + + listener.onClosed(socket, 4401, "unauthorized") + + assertFalse(fixture.transport.isConnected) + assertEquals(ConnectionStatus.AUTH_EXPIRED, fixture.transport.connectionStatus.value) + assertNull(fixture.transport.reconnectJobForTest) + assertFalse(pendingDeferred.isCompleted) + + pendingDeferred.cancel() + } + + @Test + fun disconnectPreservesPendingCallsWhileClearFlagControlsQueueAndLeaseCleanup() = + runTest { + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + every { socket.send(any()) } returns true + + fixture.transport.connectedForTest.set(true) + fixture.transport.intentionalCloseForTest.set(false) + fixture.transport.webSocketForTest = socket + + // Register actual pending RPC call over socket accepting send + val pendingDeferred = fixture.rpcChannel.request("custom.in_flight", emptyMap()) + assertFalse(pendingDeferred.isCompleted) + + // Populate retryable message queue + fixture.transport.messageQueueForTest.add("{\"jsonrpc\":\"2.0\",\"id\":\"call-1\"}") + + // Ordinary disconnect preserves queued messages and does NOT reject pending calls + fixture.transport.disconnect(clearPendingMessages = false) + assertEquals(1, fixture.transport.messageQueueForTest.size) + assertFalse(fixture.transport.isConnected) + assertFalse(pendingDeferred.isCompleted) + + // Disconnect with clearPendingMessages = true clears queue and resets leases, still not rejecting pending calls + fixture.transport.acquireBackgroundConnectionLease() + assertTrue(fixture.transport.hasBackgroundConnectionLease()) + + fixture.transport.disconnect(clearPendingMessages = true) + assertTrue(fixture.transport.messageQueueForTest.isEmpty()) + assertFalse(fixture.transport.hasBackgroundConnectionLease()) + assertFalse(pendingDeferred.isCompleted) + + pendingDeferred.cancel() + } + + @Test + fun reconnectForNetworkChangeRejectsPendingCallsAndExecutesReplacementCallbackOnce() = + runTest { + every { AuthManager.isAutoReconnect() } returns true + + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + every { socket.send(any()) } returns true + + fixture.transport.connectedForTest.set(true) + fixture.transport.intentionalCloseForTest.set(false) + fixture.transport.connectionGenerationForTest.set(1) + fixture.transport.webSocketForTest = socket + + // Register non-empty pending RPC while socket accepts send + val pendingDeferred = fixture.rpcChannel.request("custom.network_call", emptyMap()) + assertFalse(pendingDeferred.isCompleted) + + var replacementOpens = 0 + fixture.transport.reconnectForNetworkChange( + networkAvailable = true, + openReplacement = { replacementOpens++ }, + ) + + // Outgoing socket canceled once + verify(exactly = 1) { socket.cancel() } + assertEquals(1, replacementOpens) + assertEquals(2, fixture.transport.connectionGenerationForTest.get()) + assertFalse(fixture.transport.isConnected) + assertEquals(ConnectionStatus.RECONNECTING, fixture.transport.connectionStatus.value) + assertNull(fixture.transport.webSocketForTest) + + // Network replacement uses the default connection-lost exception, not a timeout. + assertTrue(pendingDeferred.isCompleted) + try { + pendingDeferred.await() + fail("Expected HermesRpcException") + } catch (exception: HermesWsClient.HermesRpcException) { + assertEquals(0, exception.code) + assertEquals("Connection lost — request cancelled", exception.message) + } + } + + @Test + fun combinedLeasesSurviveIdleBackgroundUntilBothReleased() = + runTest { + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + + fixture.transport.connectedForTest.set(true) + fixture.transport.webSocketForTest = socket + fixture.transport.setConnectionStatusForTest(ConnectionStatus.CONNECTED) + + fixture.transport.acquireExternalActivityConnectionLease() + fixture.transport.acquireBackgroundConnectionLease() + assertTrue(fixture.transport.hasBackgroundConnectionLease()) + + // Backgrounding app with both leases active keeps connection alive + fixture.transport.setAppForeground(false) + assertTrue(fixture.transport.isConnected) + + // Releasing external lease keeps connection alive due to background lease + fixture.transport.releaseExternalActivityConnectionLease() + assertTrue(fixture.transport.isConnected) + + // Releasing background lease releases all leases and disconnects idle background socket + fixture.transport.releaseBackgroundConnectionLease() + assertFalse(fixture.transport.isConnected) + assertEquals(ConnectionStatus.DISCONNECTED, fixture.transport.connectionStatus.value) + } + + @Test + fun observeEventMarksPendingReplyStartAndCompleteRespectingIdlePolicy() = + runTest { + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + + fixture.transport.connectedForTest.set(true) + fixture.transport.webSocketForTest = socket + fixture.transport.setConnectionStatusForTest(ConnectionStatus.CONNECTED) + + // When a message start event arrives BEFORE setAppForeground(false) + val startEvent = WsEvent.MessageStart(sessionId = "s1") + fixture.transport.observeEvent(startEvent) + assertTrue(fixture.transport.pendingReply) + + // Setting app foreground to false while pending reply is true stays connected + fixture.transport.setAppForeground(false) + assertTrue(fixture.transport.isConnected) + + // Idle disconnect check while pending reply is true keeps socket connected + fixture.transport.disconnectIfIdleInBackground() + assertTrue(fixture.transport.isConnected) + + // Message complete event clears pending reply and immediately idle disconnects + val completeEvent = WsEvent.MessageComplete(text = "done", sessionId = "s1") + fixture.transport.observeEvent(completeEvent) + assertFalse(fixture.transport.pendingReply) + assertFalse(fixture.transport.isConnected) + assertEquals(ConnectionStatus.DISCONNECTED, fixture.transport.connectionStatus.value) + } + + @Test + fun sendDirectFrameBypassesQueueAndReturnsFalseWhenDisconnected() = + runTest { + val fixture = Fixture(this, testJson) + val socket = mockk(relaxed = true) + every { socket.send("{\"jsonrpc\":\"2.0\",\"method\":\"connected_frame\"}") } returns true + every { socket.send("{\"jsonrpc\":\"2.0\",\"method\":\"send_fails\"}") } returns false + + fixture.transport.connectedForTest.set(true) + fixture.transport.webSocketForTest = socket + + // 1. Connected with socket accepting send -> returns true, no queue + val connectedResult = + fixture.transport.sendDirectFrame( + "{\"jsonrpc\":\"2.0\",\"method\":\"connected_frame\"}", + ) + assertTrue(connectedResult) + assertTrue(fixture.transport.messageQueueForTest.isEmpty()) + + // 2. Connected with socket rejecting send -> returns false, no queue + val sendFailsResult = fixture.transport.sendDirectFrame("{\"jsonrpc\":\"2.0\",\"method\":\"send_fails\"}") + assertFalse(sendFailsResult) + assertTrue(fixture.transport.messageQueueForTest.isEmpty()) + + // 3. Disconnected -> returns false, no queue + fixture.transport.connectedForTest.set(false) + fixture.transport.webSocketForTest = null + + val disconnectedResult = fixture.transport.sendDirectFrame("{\"jsonrpc\":\"2.0\",\"method\":\"direct\"}") + assertFalse(disconnectedResult) + assertTrue(fixture.transport.messageQueueForTest.isEmpty()) + } +}