refactor(#1376): extract WebSocket client components (#1424)

This commit is contained in:
m57 authored and GitHub committed 2026-10-01 10:35:59 +03:00
1 parent 6a75a2f1fb
commit 9cf8c3818c
16 files changed
+5285 -1913

No files matched your search

@@ -0,0 +1,296 @@
package com.m57.hermescontrol.data.ws
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.withTimeout
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonPrimitive
import okhttp3.Protocol
import okhttp3.Request
import okhttp3.Response
import okhttp3.WebSocket
import okio.ByteString
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
import org.junit.runner.RunWith
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
@RunWith(AndroidJUnit4::class)
class WsTransportDeviceTest {
private class FakeWebSocket(
private val request: Request,
var accept: Boolean = true,
var queueSizeValue: Long = 0L,
) : WebSocket {
val sentFrames = ConcurrentLinkedQueue<String>()
val attempts = AtomicInteger(0)
val closedCode = AtomicInteger(-1)
val closedReason =
java.util.concurrent.atomic
.AtomicReference<String?>(null)
val canceled = AtomicBoolean(false)
override fun request(): Request = request
override fun queueSize(): Long = queueSizeValue
override fun send(text: String): Boolean {
attempts.incrementAndGet()
if (!accept) return false
sentFrames.add(text)
return true
}
override fun send(bytes: ByteString): Boolean {
attempts.incrementAndGet()
if (!accept) return false
sentFrames.add(bytes.utf8())
return true
}
override fun close(
code: Int,
reason: String?,
): Boolean {
closedCode.set(code)
closedReason.set(reason)
return true
}
override fun cancel() {
canceled.set(true)
}
}
private class TestFixture {
val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
val lock = Any()
val clock = AtomicLong(100L)
val dummyRequest: Request = Request.Builder().url("http://localhost/ws").build()
val openResponse: Response =
Response
.Builder()
.request(dummyRequest)
.protocol(Protocol.HTTP_1_1)
.code(101)
.message("Switching Protocols")
.build()
val json =
Json {
ignoreUnknownKeys = true
encodeDefaults = false
}
val recordedEvents = ConcurrentLinkedQueue<WsEvent>()
lateinit var transport: WsTransport
lateinit var rpc: RpcChannel
lateinit var health: ConnectionHealth
lateinit var replay: ReplayTracker
init {
health =
ConnectionHealth(
scope = scope,
isConnected = { transport.isConnected },
pingRequest = { _ -> },
cancelSocket = { transport.cancelSocket() },
pendingReply = { transport.pendingReply },
nowMs = { clock.get() },
)
replay =
ReplayTracker(
scope = scope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { recordedEvents.add(it) },
)
rpc =
RpcChannel(
scope = scope,
json = json,
outboundLock = lock,
sendRequest = { method, params, onSent ->
transport.send(rpc.createRequest(method, params), onSent)
},
sendFrame = { transport.sendDirectFrame(it) },
removeQueuedMessage = { transport.removeQueuedMessage(it) },
onIdleCheck = { transport.disconnectIfIdleInBackground() },
onResult = { transport.onRpcResult(it) },
onError = { transport.onRpcError(it) },
)
transport =
WsTransport(
scope = scope,
outboundLock = lock,
reconnectPolicy = ReconnectPolicy(),
connectionHealth = health,
replayTracker = replay,
rpcChannel = rpc,
onRawMessage = { raw ->
rpc.handleIncomingFrame(raw, replay) { recordedEvents.add(it) }
},
)
}
fun close() {
try {
transport.disconnect(clearPendingMessages = true)
} finally {
scope.cancel()
}
}
}
@Test
fun transportRpcRoundTripWithStaleListenerIsolation() =
kotlinx.coroutines.runBlocking {
val fixture = TestFixture()
try {
val fakeWs = FakeWebSocket(fixture.dummyRequest, accept = true, queueSizeValue = 0L)
fixture.transport.connectionGenerationForTest.set(1)
fixture.transport.intentionalCloseForTest.set(false)
val listenerGen1 = fixture.transport.createListenerForTest(1)
listenerGen1.onOpen(fakeWs, fixture.openResponse)
assertTrue(fixture.transport.isConnected)
assertEquals(fakeWs, fixture.transport.webSocketForTest)
val deferred = fixture.rpc.request("gateway.ping", emptyMap())
val sentFrame = requireNotNull(fakeWs.sentFrames.poll())
val parsedSent = fixture.json.decodeFromString<JsonRpcRequest>(sentFrame)
val reqId = parsedSent.id
fixture.clock.set(200L)
val responseJson = """{"jsonrpc":"2.0","id":"$reqId","result":42.0}"""
listenerGen1.onMessage(fakeWs, responseJson)
val result = withTimeout(2_000L) { deferred.await() }
assertEquals(JsonPrimitive(42.0), result)
val event = requireNotNull(fixture.recordedEvents.poll())
assertTrue(event is WsEvent.RpcResult)
assertEquals(reqId, (event as WsEvent.RpcResult).id)
assertEquals(200L, fixture.health.lastPongTimestamp)
val staleListener = fixture.transport.createListenerForTest(0)
val staleWs = FakeWebSocket(fixture.dummyRequest, accept = true)
staleListener.onMessage(staleWs, """{"jsonrpc":"2.0","id":999,"result":99.0}""")
listenerGen1.onMessage(staleWs, """{"jsonrpc":"2.0","id":998,"result":88.0}""")
assertTrue(fixture.transport.isConnected)
assertEquals(fakeWs, fixture.transport.webSocketForTest)
assertEquals(200L, fixture.health.lastPongTimestamp)
assertTrue(fixture.recordedEvents.isEmpty())
} finally {
fixture.close()
}
}
@Test
fun rejectingWsQueuesAndFlushesOnNextAcceptSocket() =
kotlinx.coroutines.runBlocking {
val fixture = TestFixture()
try {
val rejectingWs = FakeWebSocket(fixture.dummyRequest, accept = false, queueSizeValue = 1L)
fixture.transport.connectionGenerationForTest.set(1)
fixture.transport.intentionalCloseForTest.set(false)
val listener1 = fixture.transport.createListenerForTest(1)
listener1.onOpen(rejectingWs, fixture.openResponse)
assertTrue(fixture.transport.isConnected)
var recordedId: String? = null
var callbackRanUnderLock = false
val req1 = fixture.rpc.createRequest("test.one", emptyMap())
val returnedId =
fixture.transport.send(
request = req1,
onSent = { id ->
callbackRanUnderLock = Thread.holdsLock(fixture.lock)
recordedId = id
},
)
assertEquals(returnedId, req1.id.toString())
assertEquals(returnedId, recordedId)
assertTrue(callbackRanUnderLock)
val req2 = fixture.rpc.createRequest("test.two", emptyMap())
fixture.transport.messageQueueForTest.add(fixture.rpc.encodeRequest(req2))
assertFalse(fixture.transport.isConnected)
assertEquals(ConnectionStatus.RECONNECTING, fixture.transport.connectionStatus.value)
assertNotNull(fixture.transport.outboundDrainJobForTest)
assertEquals(2, fixture.transport.messageQueueForTest.size)
val acceptWs = FakeWebSocket(fixture.dummyRequest, accept = true, queueSizeValue = 0L)
val listenerReplacement = fixture.transport.createListenerForTest(1)
listenerReplacement.onOpen(acceptWs, fixture.openResponse)
assertTrue(fixture.transport.isConnected)
assertEquals(ConnectionStatus.CONNECTED, fixture.transport.connectionStatus.value)
assertNull(fixture.transport.outboundDrainJobForTest)
assertTrue(fixture.transport.messageQueueForTest.isEmpty())
val frameList = acceptWs.sentFrames.toList()
assertEquals(
listOf(fixture.rpc.encodeRequest(req1), fixture.rpc.encodeRequest(req2)),
frameList,
)
} finally {
fixture.close()
}
}
@Test
fun authClosingDoesNotPrematurelyDropPendingRpcAndClosedHaltsReconnect() =
kotlinx.coroutines.runBlocking {
val fixture = TestFixture()
try {
val fakeWs = FakeWebSocket(fixture.dummyRequest, accept = true, queueSizeValue = 0L)
fixture.transport.connectionGenerationForTest.set(1)
fixture.transport.intentionalCloseForTest.set(false)
val listener = fixture.transport.createListenerForTest(1)
listener.onOpen(fakeWs, fixture.openResponse)
assertTrue(fixture.transport.isConnected)
val pendingDeferred = fixture.rpc.request("gateway.ping", emptyMap())
assertTrue(fixture.rpc.hasPendingCalls())
listener.onClosing(fakeWs, 4401, "AUTH_EXPIRED")
assertTrue(fixture.transport.isConnected)
assertTrue(fixture.rpc.hasPendingCalls())
assertFalse(pendingDeferred.isCompleted)
listener.onClosed(fakeWs, 4401, "AUTH_EXPIRED")
assertFalse(fixture.transport.isConnected)
assertNull(fixture.transport.reconnectJobForTest)
assertTrue(fixture.rpc.hasPendingCalls())
assertFalse(pendingDeferred.isCompleted)
fixture.transport.disconnect(clearPendingMessages = true)
assertFalse(pendingDeferred.isCompleted)
pendingDeferred.cancel()
} finally {
fixture.close()
}
}
}
@@ -0,0 +1,140 @@
package com.m57.hermescontrol.data.ws
import androidx.annotation.VisibleForTesting
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
internal class ConnectionHealth(
private val scope: CoroutineScope,
private val isConnected: () -> Boolean,
private val pingRequest: suspend (Long) -> Unit,
private val cancelSocket: () -> Unit,
private val pendingReply: () -> Boolean,
private val nowMs: () -> Long = { System.nanoTime() / 1_000_000L },
private val log: (String) -> Unit = {},
private val debugLog: (String) -> Unit = {},
) {
companion object {
internal const val HEARTBEAT_INTERVAL_MS = 15_000L
internal const val STALE_THRESHOLD_MS = 30_000L
internal const val LIVENESS_PROBE_TIMEOUT_MS = 5_000L
}
@Volatile
var lastPongTimestamp: Long = 0L
internal set
private val _lastLatencyMs = MutableStateFlow<Long?>(null)
val lastLatencyMs: StateFlow<Long?> = _lastLatencyMs.asStateFlow()
val isHealthy: Boolean
get() = isConnected() && (nowMs() - lastPongTimestamp < STALE_THRESHOLD_MS)
private var healthJob: Job? = null
private var foregroundProbeJob: Job? = null
private var transportProbeJob: Job? = null
fun start() {
healthJob?.cancel()
lastPongTimestamp = nowMs()
healthJob =
scope.launch {
while (isConnected()) {
delay(HEARTBEAT_INTERVAL_MS)
if (isConnected()) {
try {
ping(timeoutMs = LIVENESS_PROBE_TIMEOUT_MS)
} catch (e: Exception) {
log("Health check ping failed: ${e.message}")
}
}
if (!runHealthCheckPass()) break
}
}
}
fun stop() {
healthJob?.cancel()
healthJob = null
_lastLatencyMs.value = null
}
fun onInboundFrame() {
lastPongTimestamp = nowMs()
}
suspend fun ping(timeoutMs: Long = LIVENESS_PROBE_TIMEOUT_MS): Long {
val start = nowMs()
pingRequest(timeoutMs)
val latency = (nowMs() - start).coerceAtLeast(0L)
lastPongTimestamp = nowMs()
_lastLatencyMs.value = latency
return latency
}
@VisibleForTesting
internal fun setLastLatencyForTest(latency: Long?) {
_lastLatencyMs.value = latency
}
private fun runHealthCheckPass(): Boolean {
if (!isConnected()) return false
val staleMs = nowMs() - lastPongTimestamp
if (staleMs <= STALE_THRESHOLD_MS) return true
log("WebSocket stale (${staleMs / 1000}s without frames) — cancelling to trigger reconnect")
cancelSocket()
return false
}
@VisibleForTesting
internal fun forceHealthCheckForTest(staleMillis: Long) {
lastPongTimestamp = nowMs() - staleMillis
runHealthCheckPass()
}
fun probeLivenessOnWake(deferredOnce: Boolean = false) {
if (!isConnected()) return
foregroundProbeJob?.cancel()
foregroundProbeJob =
scope.launch {
val alive = runCatching { ping(LIVENESS_PROBE_TIMEOUT_MS) }.isSuccess
if (alive) return@launch
if (pendingReply() && !deferredOnce) {
delay(3_000L)
probeLivenessOnWake(deferredOnce = true)
return@launch
}
log("Foreground liveness probe failed — cancelling socket")
cancelSocket()
}
}
fun cancelForegroundProbe() {
foregroundProbeJob?.cancel()
foregroundProbeJob = null
}
fun probeLivenessOnTransportChange(
timeoutMs: Long = LIVENESS_PROBE_TIMEOUT_MS,
onFailureAction: () -> Unit = cancelSocket,
) {
if (!isConnected()) return
transportProbeJob?.cancel()
transportProbeJob =
scope.launch {
debugLog("Network transport changed — probing WebSocket liveness")
val alive = runCatching { ping(timeoutMs) }.isSuccess
if (alive) {
debugLog("WebSocket liveness probe succeeded on new transport")
return@launch
}
log("Transport change liveness probe failed — cancelling socket")
onFailureAction()
}
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,39 @@
package com.m57.hermescontrol.data.ws
internal class ReconnectPolicy(
private val random: () -> Double = { Math.random() },
) {
@Volatile
var currentBackoff: Long = INITIAL_BACKOFF_MS
private var initialBackoffMs: Long = INITIAL_BACKOFF_MS
private var testBackoffOverrideMs: Long? = null
fun nextDelayMs(): Long {
val override = testBackoffOverrideMs
if (override != null) return override
val ceiling = currentBackoff
if (ceiling <= 0L) return 0L
return (random() * ceiling).toLong().coerceAtLeast(BACKOFF_BASE_MS).coerceAtMost(ceiling)
}
fun advance() {
currentBackoff = (currentBackoff * BACKOFF_MULTIPLIER).toLong().coerceAtMost(MAX_BACKOFF_MS)
}
fun reset() {
currentBackoff = initialBackoffMs
}
fun setBackoffForTest(initialMillis: Long) {
initialBackoffMs = initialMillis
currentBackoff = initialMillis
testBackoffOverrideMs = initialMillis
}
companion object {
private const val INITIAL_BACKOFF_MS = 1_000L
private const val BACKOFF_BASE_MS = 300L
private const val MAX_BACKOFF_MS = 30_000L
private const val BACKOFF_MULTIPLIER = 2.0
}
}
@@ -0,0 +1,245 @@
package com.m57.hermescontrol.data.ws
import androidx.annotation.VisibleForTesting
import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceParams
import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.JsonObject
import java.util.Collections
import java.util.concurrent.ConcurrentHashMap
/**
* Tracks session event sequence watermarks and orchestrates event replay on reconnect.
*
* Extracted from [HermesWsClient] (issue #1016, issue #1376).
*/
internal class ReplayTracker(
private val scope: CoroutineScope,
private val fetchEventsSince: suspend (SessionEventsSinceParams) -> SessionEventsSinceResult,
private val emitEvent: (WsEvent) -> Unit,
private val resolveStoredSessionId: (String?) -> String? = { it },
private val onReplayFailure: (String, Exception) -> Unit = { _, _ -> },
) {
private val lastSeenSeq = ConcurrentHashMap<String, Int>()
@Volatile
private var replayEpoch: String? = null
@Volatile
private var replayInFlight: Boolean = false
private val replayHold = ConcurrentHashMap<String, MutableList<Pair<Int?, WsEvent>>>()
@Volatile
private var replayJob: Job? = null
/**
* Synchronously arms the hold buffers for all known sessions and launches replay in [scope].
* Arming synchronously ensures racing live frames are held instead of advancing watermarks.
*/
fun triggerReplay() {
if (lastSeenSeq.isEmpty() || replayInFlight) return
replayInFlight = true
val sessionsToReplay = lastSeenSeq.keys().toList()
for (sid in sessionsToReplay) {
replayHold[sid] = Collections.synchronizedList(mutableListOf())
}
replayJob?.cancel()
replayJob =
scope.launch {
fetchReplay(sessionsToReplay)
}
}
/**
* Updates replay epoch from gateway.ready.
* When [epoch] is non-empty and changes from a previous non-null value, resets sequence watermarks.
*/
fun updateEpoch(epoch: String?) {
if (!epoch.isNullOrEmpty()) {
if (replayEpoch != null && replayEpoch != epoch) {
lastSeenSeq.clear()
}
replayEpoch = epoch
}
}
/**
* Checks if an incoming event is a duplicate against the current watermark for [sessionId].
* Returns true if [seq] <= watermark.
*/
fun isDuplicate(
sessionId: String,
seq: Int,
): Boolean {
val prev = lastSeenSeq[sessionId] ?: 0
return seq <= prev
}
/**
* Processes an incoming live event with sequence tracking.
* Evaluates [eventFactory] only after checking for duplicates.
* If duplicate, drops it without calling [eventFactory].
* If replay is in flight for [sessionId], buffers it in replayHold.
* Otherwise advances watermark and emits the event.
*/
fun acceptLiveEvent(
sessionId: String,
seq: Int,
eventFactory: () -> WsEvent,
) {
if (isDuplicate(sessionId, seq)) {
return
}
val finalEvent = decorateEvent(eventFactory())
if (replayInFlight && replayHold.containsKey(sessionId)) {
replayHold[sessionId]?.add(seq to finalEvent)
return
}
lastSeenSeq[sessionId] = seq
emitEvent(finalEvent)
}
/**
* Processes an incoming live event with sequence tracking.
* If duplicate, drops it.
* If replay is in flight for [sessionId], buffers it in replayHold.
* Otherwise advances watermark and emits the event.
*/
fun acceptLiveEvent(
sessionId: String,
seq: Int,
event: WsEvent,
) {
acceptLiveEvent(sessionId, seq) { event }
}
private fun decorateEvent(event: WsEvent): WsEvent =
if (event is WsEvent.MessageComplete) {
event.copy(
storedSessionId = resolveStoredSessionId(event.sessionId),
)
} else {
event
}
private suspend fun fetchReplay(sessionsToReplay: List<String> = lastSeenSeq.keys().toList()) {
try {
for (sid in sessionsToReplay) {
val lastSeen = lastSeenSeq[sid] ?: continue
try {
val res =
fetchEventsSince(
SessionEventsSinceParams(sessionId = sid, lastSeen = lastSeen),
)
val epoch = res.epoch
val epochChanged =
!epoch.isNullOrEmpty() && replayEpoch != null && replayEpoch != epoch
val truncated = res.truncated == true
val latestSeq = res.latestSeq
if (epochChanged) replayEpoch = epoch
if (epochChanged || truncated) {
// Ring buffer (512 events) could not cover the gap or epoch changed (gateway restarted).
// Partial replay would silently hole the transcript — fast-forward watermark
// and request full transcript resync.
if (epochChanged) lastSeenSeq.clear()
if (latestSeq != null && !epochChanged) lastSeenSeq[sid] = latestSeq
emitEvent(WsEvent.TranscriptResyncRequired(sid))
continue
}
val eventsList = res.events
if (eventsList != null) {
for (element in eventsList) {
@Suppress("UNCHECKED_CAST")
val eventMap = (element as? JsonObject)?.toAny() as? Map<String, Any?> ?: continue
val eventSeq = (eventMap["seq"] as? Number)?.toInt()
val eventSid = (eventMap["session_id"] as? String) ?: sid
if (eventSeq != null && eventSid.isNotEmpty()) {
val prev = lastSeenSeq[eventSid] ?: 0
if (eventSeq <= prev) continue
lastSeenSeq[eventSid] = eventSeq
}
val parsedEvent = EventParser.parseParams(eventMap)
val finalEvent = decorateEvent(parsedEvent)
emitEvent(finalEvent)
}
}
} catch (e: Exception) {
onReplayFailure(sid, e)
emitEvent(WsEvent.TranscriptResyncRequired(sid))
}
}
} finally {
withContext(NonCancellable) {
for (sid in sessionsToReplay) {
val held = replayHold.remove(sid)
if (held != null) {
synchronized(held) {
for ((heldSeq, heldEvent) in held) {
if (heldSeq != null) {
val prev = lastSeenSeq[sid] ?: 0
if (heldSeq <= prev) continue
lastSeenSeq[sid] = heldSeq
}
emitEvent(heldEvent)
}
}
}
}
replayInFlight = false
}
}
}
/**
* Complete teardown matching disconnect (issue #1016).
* Cancels any active replay job and clears all holds, watermarks, and epoch.
*/
fun clear() {
lastSeenSeq.clear()
replayHold.clear()
replayEpoch = null
replayJob?.cancel()
replayJob = null
replayInFlight = false
}
@VisibleForTesting
internal fun getSeqWatermarks(): Map<String, Int> = HashMap(lastSeenSeq)
@VisibleForTesting
internal fun setSeqWatermark(
sessionId: String,
seq: Int,
) {
lastSeenSeq[sessionId] = seq
}
@VisibleForTesting
internal fun clearSeqWatermarks() {
lastSeenSeq.clear()
replayHold.clear()
replayEpoch = null
replayInFlight = false
}
@VisibleForTesting
internal fun isReplayInFlight(): Boolean = replayInFlight
@VisibleForTesting
internal fun setReplayEpochForTest(epoch: String?) {
replayEpoch = epoch
}
@VisibleForTesting
internal suspend fun fetchReplayForTest() {
fetchReplay()
}
}
@@ -0,0 +1,353 @@
package com.m57.hermescontrol.data.ws
import com.m57.hermescontrol.data.ws.contract.RpcMethod
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.put
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicInteger
/**
* Socket-independent JSON-RPC 2.0 request correlation and response dispatcher.
* Manages request IDs, in-flight pending calls, timeouts, cancellation cleanup,
* and server response frame formatting.
*/
internal class RpcChannel(
private val scope: CoroutineScope,
private val json: Json,
private val sendRequest: (method: String, params: Map<String, Any>, onSent: ((String) -> Unit)?) -> String,
private val sendFrame: (String) -> Boolean,
private val removeQueuedMessage: (String) -> Unit,
private val onIdleCheck: () -> Unit,
private val logRejected: (method: String, id: String) -> Unit = { _, _ -> },
private val outboundLock: Any = Any(),
private val onResult: (String) -> Unit = {},
private val onError: (String) -> Unit = {},
private val resolveStoredSessionId: (String?) -> String? = { it },
private val logParseFailure: (Exception) -> Unit = {},
private val decorateParams: (String, Map<String, Any>) -> Map<String, Any> = { _, p -> p },
) {
/** A single in-flight [request] awaiting its RPC result/error. */
private data class PendingCall(
val method: String,
val deferred: CompletableDeferred<Any?>,
var timeoutJob: Job? = null,
val suppressErrorEvent: Boolean = false,
)
private val requestId = AtomicInteger(0)
private val pendingCalls = ConcurrentHashMap<String, PendingCall>()
private val capabilityRequestIds = ConcurrentHashMap.newKeySet<String>()
fun nextId(): String = requestId.incrementAndGet().toString()
fun hasPendingCalls(): Boolean = pendingCalls.isNotEmpty()
fun pendingCallIdsForTest(): Set<String> = pendingCalls.keys.toSet()
fun pendingCallTimeoutForTest(id: String): Job? = pendingCalls[id]?.timeoutJob
fun isErrorEventSuppressed(id: String): Boolean = pendingCalls[id]?.suppressErrorEvent == true
fun registerCapabilityRequest(id: String) {
capabilityRequestIds.add(id)
}
fun clearCapabilityRequests() {
capabilityRequestIds.clear()
}
fun consumeCapabilityResponse(event: WsEvent): Boolean {
val id =
when (event) {
is WsEvent.RpcResult -> event.id
is WsEvent.RpcError -> event.id
else -> return false
}
val isCapabilityResponse =
synchronized(outboundLock) {
capabilityRequestIds.remove(id)
}
if (!isCapabilityResponse) return false
removeQueuedMessage(id)
return true
}
fun createRequest(
method: String,
params: Map<String, Any>,
): JsonRpcRequest {
val id = nextId()
val decorated = decorateParams(method, params)
return JsonRpcRequest(
id = id,
method = method,
params = decorated.mapValues { it.value.toJsonElement() },
)
}
fun encodeRequest(request: JsonRpcRequest): String = json.encodeToString(request)
/**
* Send a JSON-RPC request that expects a result and returns a [CompletableDeferred] for it.
*/
fun request(
method: String,
params: Map<String, Any> = emptyMap(),
timeoutMs: Long = HermesWsClient.REQUEST_TIMEOUT_MS,
suppressErrorEvent: Boolean = false,
): CompletableDeferred<Any?> {
val deferred = CompletableDeferred<Any?>()
val id =
sendRequest(method, params) { reqId ->
pendingCalls[reqId] = PendingCall(method, deferred, suppressErrorEvent = suppressErrorEvent)
}
deferred.invokeOnCompletion { cause ->
if (cause is CancellationException) {
pendingCalls.remove(id)?.let { call ->
call.timeoutJob?.cancel()
removeQueuedMessage(id)
onIdleCheck()
}
}
}
// Arm the per-request timeout (fires if the server never answers).
pendingCalls[id]?.timeoutJob =
scope.launch {
delay(timeoutMs)
resolvePending(id, null, JsonRpcError(-1, "Request timed out: $method"))
}
return deferred
}
/**
* Non-suspending typed twin of [request] for callers that must enqueue inside a
* non-suspending owner check and await the raw result themselves.
*/
fun <P, R> requestTyped(
method: RpcMethod<P, R>,
params: P,
timeoutMs: Long = HermesWsClient.REQUEST_TIMEOUT_MS,
suppressErrorEvent: Boolean = false,
): CompletableDeferred<Any?> = request(method.name, encodeParams(method, params), timeoutMs, suppressErrorEvent)
/**
* Send a typed JSON-RPC request and await its deserialized result.
*/
suspend fun <P, R> call(
method: RpcMethod<P, R>,
params: P,
timeoutMs: Long = HermesWsClient.REQUEST_TIMEOUT_MS,
suppressErrorEvent: Boolean = false,
): R = awaitResult(method, requestTyped(method, params, timeoutMs, suppressErrorEvent))
suspend fun <P, R> awaitResult(
method: RpcMethod<P, R>,
deferred: CompletableDeferred<Any?>,
): R {
val result =
try {
deferred.await()
} catch (e: CancellationException) {
deferred.cancel(e)
throw e
}
val element = result.toJsonElement()
return json.decodeFromJsonElement(method.result, element)
}
fun <P, R> encodeParams(
method: RpcMethod<P, R>,
params: P,
): JsonObject {
val encoded = json.encodeToJsonElement(method.params, params)
require(encoded is JsonObject) {
"RPC params for ${method.name} must serialize to a JsonObject, got ${encoded::class.simpleName}"
}
return encoded
}
/** Complete (or fail) a single pending call and cancel its timer. */
fun resolvePending(
id: String,
result: Any?,
error: JsonRpcError?,
) {
val call = pendingCalls.remove(id) ?: return
removeQueuedMessage(id)
call.timeoutJob?.cancel()
if (error != null) {
call.deferred.completeExceptionally(
HermesWsClient.HermesRpcException(
message = error.message,
code = error.code,
data = error.data,
),
)
} else {
call.deferred.complete(result)
}
onIdleCheck()
}
/**
* Fail and clear every in-flight [request].
*/
fun rejectAllPending(
error: HermesWsClient.HermesRpcException =
HermesWsClient.HermesRpcException("Connection lost — request cancelled"),
) {
if (pendingCalls.isEmpty()) return
val snapshot = pendingCalls.toList()
pendingCalls.clear()
for ((id, call) in snapshot) {
removeQueuedMessage(id)
logRejected(call.method, id)
call.timeoutJob?.cancel()
call.deferred.completeExceptionally(error)
}
}
/**
* Answer a gateway-originated JSON-RPC request.
*/
fun respondToServerRequest(
id: String,
result: JsonElement,
): Boolean {
if (id.isBlank()) return false
val frameJson =
buildJsonObject {
put("jsonrpc", "2.0")
put("id", id)
put("result", result)
}.toString()
return sendFrame(frameJson)
}
/** Answer a gateway-originated request with a standard JSON-RPC error. */
fun respondToServerRequestError(
id: String,
code: Int,
message: String,
): Boolean {
if (id.isBlank()) return false
val frameJson =
buildJsonObject {
put("jsonrpc", "2.0")
put("id", id)
put(
"error",
buildJsonObject {
put("code", code)
put("message", message)
},
)
}.toString()
return sendFrame(frameJson)
}
/**
* Handles an incoming WebSocket text frame: JSON decoding, request correlation,
* server request dispatch, sequence deduplication via [replayTracker], and event emission.
*/
fun handleIncomingFrame(
text: String,
replayTracker: ReplayTracker,
emitEvent: (WsEvent) -> Unit,
) {
val event =
try {
val rpc = json.decodeFromString<JsonRpcResponse>(text)
// Resolve pending request() calls with the RAW JsonElement from
// rpc.result, BEFORE EventParser.parse() converts it via toAny().
// This preserves integer/float type fidelity needed by callers
// that re-deserialize the result into typed data classes.
val rpcId = rpc.id
if (rpcId != null && rpc.error == null && rpc.result != null) {
onResult(rpcId)
removeQueuedMessage(rpcId)
resolvePending(rpcId, rpc.result, null)
}
// Resume/replay responses carry still-open server requests
// separately from the event ring. Re-deliver them through the
// same generic path before exposing the RPC result.
@Suppress("UNCHECKED_CAST")
val openRequests =
(rpc.result?.toAny() as? Map<*, *>)?.get("open_requests") as? List<Map<*, *>>
openRequests?.forEach { openRequest ->
val openId = openRequest["id"] as? String ?: return@forEach
val method = openRequest["method"] as? String ?: return@forEach
@Suppress("UNCHECKED_CAST")
val params = openRequest["params"] as? Map<String, Any?> ?: emptyMap()
emitEvent(WsEvent.ServerRequest(openId, method, params, replayed = true))
}
if (rpc.id == null) {
// Issue #1163: inspect scalars, not a throwaway copy of the whole params tree.
val sid = rpc.params?.eventSessionId()
val seq = ((rpc.params?.get("seq") as? JsonPrimitive)?.toAny() as? Number)?.toInt()
if (!sid.isNullOrBlank() && seq != null) {
replayTracker.acceptLiveEvent(sid, seq) {
EventParser.parse(rpc, text)
}
return
}
}
val parsed = EventParser.parse(rpc, text)
if (parsed is WsEvent.MessageComplete) {
parsed.copy(
storedSessionId = resolveStoredSessionId(parsed.sessionId),
)
} else {
parsed
}
} catch (e: Exception) {
logParseFailure(e)
WsEvent.Unknown(text)
}
if (consumeCapabilityResponse(event)) return
when (event) {
is WsEvent.RpcResult -> {
onResult(event.id)
removeQueuedMessage(event.id)
resolvePending(event.id, event.result, null)
}
is WsEvent.RpcError -> {
onError(event.id)
if (pendingCalls[event.id]?.suppressErrorEvent == true) {
// Opt-in suppression: the caller already handles the
// failure through the CompletableDeferred, so the event
// copy would only surface a duplicate UI banner for
// optional features (e.g. "subagent.list" on gateways
// without the method, issue #1089). Default request()
// behavior still emits the event for shared consumers.
resolvePending(event.id, null, event.error)
return
}
removeQueuedMessage(event.id)
resolvePending(event.id, null, event.error)
}
else -> {}
}
emitEvent(event)
}
}
@@ -0,0 +1,912 @@
package com.m57.hermescontrol.data.ws
import android.util.Log
import androidx.annotation.VisibleForTesting
import com.m57.hermescontrol.BuildConfig
import com.m57.hermescontrol.data.local.AuthManager
import com.m57.hermescontrol.data.remote.CookieManager
import com.m57.hermescontrol.data.remote.DashboardSessionTokenRefresher
import com.m57.hermescontrol.data.remote.NetworkMonitor
import com.m57.hermescontrol.data.remote.OkHttpProvider
import com.m57.hermescontrol.data.remote.await
import com.m57.hermescontrol.data.session.ActiveSessionHolder
import com.m57.hermescontrol.data.ws.contract.PromptSubmitParams
import com.m57.hermescontrol.data.ws.contract.RpcMethods
import com.m57.hermescontrol.data.ws.contract.SessionCorrectionParams
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import okhttp3.WebSocket
import okhttp3.WebSocketListener
import okio.utf8Size
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
/**
* Socket lifecycle, authentication, outbound queueing, and network transport management.
*
* Extracted from [HermesWsClient] (issue #1376).
*/
internal class WsTransport(
private val scope: CoroutineScope,
private val outboundLock: Any,
private val reconnectPolicy: ReconnectPolicy,
private val connectionHealth: ConnectionHealth,
private val replayTracker: ReplayTracker,
private val rpcChannel: RpcChannel,
private val onRawMessage: (String) -> Unit,
) {
companion object {
private const val TAG = "HermesWsClient"
private const val MAX_OUTBOUND_MESSAGE_BYTES = 16 * 1024 * 1024
private const val OUTBOUND_DRAIN_TIMEOUT_MS = 60_000L
}
private val connectionGeneration = AtomicInteger(0)
private val connected = AtomicBoolean(false)
private val intentionalClose = AtomicBoolean(false)
private val acceptQueuedMessages = AtomicBoolean(true)
private val appInForeground = AtomicBoolean(true)
private val externalActivityConnectionLease = AtomicBoolean(false)
private val backgroundConnectionLease = AtomicBoolean(false)
private val messageQueue = ConcurrentLinkedQueue<String>()
private val queuedMessagesById = ConcurrentHashMap<String, String>()
@Volatile
private var webSocket: WebSocket? = null
@Volatile
private var closingSocket: WebSocket? = null
@Volatile
private var reconnectJob: Job? = null
@Volatile
private var outboundDrainJob: Job? = null
private val networkCollectorStarted = AtomicBoolean(false)
private val _connectionStatus = MutableStateFlow(ConnectionStatus.DISCONNECTED)
val connectionStatus: StateFlow<ConnectionStatus> = _connectionStatus.asStateFlow()
@Volatile
var pendingReply: Boolean = false
internal set
private val pendingPromptSubmits = ConcurrentHashMap.newKeySet<String>()
val isConnected: Boolean get() = connected.get()
fun setAppForeground(foreground: Boolean) {
appInForeground.set(foreground)
if (!foreground) {
disconnectIfIdleInBackground()
return
}
connectionHealth.probeLivenessOnWake()
}
fun acquireExternalActivityConnectionLease() {
externalActivityConnectionLease.set(true)
}
fun releaseExternalActivityConnectionLease() {
externalActivityConnectionLease.set(false)
disconnectIfIdleInBackground()
}
fun acquireBackgroundConnectionLease() {
backgroundConnectionLease.set(true)
}
fun releaseBackgroundConnectionLease() {
backgroundConnectionLease.set(false)
disconnectIfIdleInBackground()
}
internal fun hasBackgroundConnectionLease(): Boolean = backgroundConnectionLease.get()
fun disconnectIfIdleInBackground() {
synchronized(outboundLock) {
if (!appInForeground.get() &&
!externalActivityConnectionLease.get() &&
!backgroundConnectionLease.get() &&
!pendingReply &&
!rpcChannel.hasPendingCalls() &&
messageQueue.isEmpty()
) {
disconnect()
}
}
}
fun connect() {
if (networkCollectorStarted.compareAndSet(false, true)) {
scope.launch {
NetworkMonitor.networkChanges.collect { networkAvailable ->
reconnectForNetworkChange(networkAvailable)
}
}
scope.launch {
NetworkMonitor.transportChanges.collect {
connectionHealth.probeLivenessOnTransportChange(
onFailureAction = { cancelSocket() },
)
}
}
}
var socketToCancel: WebSocket? = null
val shouldOpen =
synchronized(outboundLock) {
acceptQueuedMessages.set(true)
if (_connectionStatus.value == ConnectionStatus.AUTH_EXPIRED) {
connectionGeneration.incrementAndGet()
reconnectJob?.cancel()
reconnectJob = null
outboundDrainJob?.cancel()
outboundDrainJob = null
socketToCancel = webSocket
webSocket = null
closingSocket = null
connected.set(false)
connectionHealth.stop()
}
if (connected.get()) {
Log.d(TAG, "Already connected — skipping")
return@synchronized false
}
if (_connectionStatus.value == ConnectionStatus.CONNECTING ||
_connectionStatus.value == ConnectionStatus.RECONNECTING
) {
Log.d(TAG, "Connection already in flight (${_connectionStatus.value}) — skipping")
return@synchronized false
}
// An explicit connect follows successful authentication and is
// therefore allowed to leave the terminal AUTH_EXPIRED state.
intentionalClose.set(false)
_connectionStatus.value = ConnectionStatus.CONNECTING
reconnectPolicy.reset()
true
}
if (socketToCancel != null) {
rpcChannel.rejectAllPending()
socketToCancel.cancel()
}
if (shouldOpen) openSocket()
}
fun reconnectForNetworkChange(
networkAvailable: Boolean = NetworkMonitor.isConnected.value,
openReplacement: () -> Unit = ::openSocket,
) {
val socketToCancel: WebSocket?
val shouldOpen: Boolean
synchronized(outboundLock) {
if (intentionalClose.get() || _connectionStatus.value == ConnectionStatus.AUTH_EXPIRED) {
return
}
Log.d(TAG, "Default network changed — reconnecting WebSocket")
reconnectPolicy.reset()
reconnectJob?.cancel()
reconnectJob = null
outboundDrainJob?.cancel()
outboundDrainJob = null
connectionGeneration.incrementAndGet()
socketToCancel = webSocket
webSocket = null
closingSocket = null
connected.set(false)
connectionHealth.stop()
shouldOpen = networkAvailable && AuthManager.isAutoReconnect()
_connectionStatus.value =
when {
!networkAvailable -> ConnectionStatus.NO_NETWORK
shouldOpen -> ConnectionStatus.RECONNECTING
else -> ConnectionStatus.DISCONNECTED
}
rpcChannel.rejectAllPending()
}
socketToCancel?.cancel()
if (shouldOpen) openReplacement()
}
suspend fun refreshWsTicketIfNeeded(generation: Int): Boolean {
val isGated =
try {
AuthManager.serverStore.getLatestState().wsAuthParam == "ticket"
} catch (_: IllegalStateException) {
// serverStore not initialized yet (transient early call); treat
// as loopback so a stale-token refresh still runs. Log so a real
// misconfiguration isn't silently swallowed.
Log.w(TAG, "serverStore uninitialized during WS handshake; assuming non-gated")
false
}
if (!isGated) {
// The loopback dashboard token is regenerated on every server
// restart. Refresh it before each WebSocket handshake so automatic
// reconnect does not get stuck in AUTH_EXPIRED with a stale token.
val token =
runCatching {
DashboardSessionTokenRefresher.refreshAsync()
}.getOrNull()
synchronized(outboundLock) {
if (isCurrentGeneration(generation) && token != null) {
runCatching { AuthManager.setToken(token) }
}
}
return true
}
val ticketResult = requestWsTicket()
if (!ticketResult.ticket.isNullOrBlank()) {
synchronized(outboundLock) {
if (!isCurrentGeneration(generation)) return false
AuthManager.setToken(ticketResult.ticket)
}
if (BuildConfig.DEBUG) Log.d(TAG, "WS ticket refreshed")
return true
}
val status =
when {
ticketResult.exception -> ConnectionStatus.RECONNECTING
ticketResult.httpCode == 401 || ticketResult.httpCode == 403 -> ConnectionStatus.AUTH_EXPIRED
ticketResult.httpCode == 408 || ticketResult.httpCode == 429 -> ConnectionStatus.RECONNECTING
ticketResult.httpCode != null && ticketResult.httpCode >= 500 -> ConnectionStatus.RECONNECTING
else -> ConnectionStatus.DISCONNECTED
}
return handleWsTicketRefreshFailure(status, generation)
}
suspend fun mintWsTicket(): String? {
val isGated =
try {
AuthManager.serverStore.getLatestState().wsAuthParam == "ticket"
} catch (_: IllegalStateException) {
false
}
if (!isGated) {
DashboardSessionTokenRefresher.refreshAsync()
return AuthManager.getToken()
}
return requestWsTicket().ticket
}
private data class TicketRequestResult(
val ticket: String?,
val httpCode: Int?,
val exception: Boolean = false,
)
private suspend fun requestWsTicket(): TicketRequestResult {
try {
val client = OkHttpProvider.probe
val request =
Request
.Builder()
.url(AuthManager.endpointForBuild().resolve("api/auth/ws-ticket").toString())
.post("{}".toRequestBody())
.build()
val (code, body) =
withContext(Dispatchers.IO) {
if (CookieManager.isInitialized()) {
CookieManager.useStore(CookieManager.cookieJar.currentServer())
}
client.newCall(request).await().use { resp ->
resp.code to resp.body.string()
}
}
if (code in 200..299) {
// Real JSON decode — the ticket is base64url and may contain
// characters a naive [^"]+ regex cannot extract (see
// testGatedMode_parsesEscapedTicketFromRealJsonShape).
val parsed =
runCatching {
OkHttpProvider.json.decodeFromString<WsTicketResponse>(body)
}.getOrNull()
val ticket = parsed?.ticket
if (!ticket.isNullOrBlank()) {
return TicketRequestResult(ticket, null)
}
Log.w(TAG, "WS ticket mint failed: unparseable ticket response")
} else {
Log.w(TAG, "WS ticket mint failed: HTTP $code")
return TicketRequestResult(null, code)
}
} catch (e: Exception) {
// Include the stack: the exception class alone (e.g.
// NetworkOnMainThreadException) can't show WHICH caller on what
// thread tripped the mint (main-scope kanban events connect vs
// the WS reconnect path).
Log.w(TAG, "WS ticket mint failed: ${e.javaClass.simpleName}", e)
return TicketRequestResult(null, null, exception = true)
}
return TicketRequestResult(null, null)
}
private fun handleWsTicketRefreshFailure(
status: ConnectionStatus,
generation: Int,
): Boolean {
synchronized(outboundLock) {
if (!isCurrentGeneration(generation)) return false
_connectionStatus.value = status
if (status == ConnectionStatus.RECONNECTING) scheduleReconnect()
}
return false
}
fun disconnect(clearPendingMessages: Boolean = false) {
ActiveSessionHolder.clear()
synchronized(outboundLock) {
intentionalClose.set(true)
acceptQueuedMessages.set(!clearPendingMessages)
connectionGeneration.incrementAndGet()
reconnectJob?.cancel()
reconnectJob = null
outboundDrainJob?.cancel()
outboundDrainJob = null
connectionHealth.cancelForegroundProbe()
connectionHealth.stop()
webSocket?.close(1000, "Client closed")
webSocket = null
closingSocket = null
rpcChannel.clearCapabilityRequests()
if (clearPendingMessages) {
backgroundConnectionLease.set(false)
messageQueue.clear()
queuedMessagesById.clear()
pendingPromptSubmits.clear()
pendingReply = false
replayTracker.clear()
}
connected.set(false)
_connectionStatus.value = ConnectionStatus.DISCONNECTED
}
}
fun cancelSocket() {
synchronized(outboundLock) {
if (connected.get()) webSocket?.cancel()
}
}
fun sendDirectFrame(frameJson: String): Boolean =
synchronized(outboundLock) {
val ws = webSocket
if (ws == null || !connected.get()) false else ws.send(frameJson)
}
fun send(
request: JsonRpcRequest,
onSent: ((String) -> Unit)? = null,
queueIfDisconnected: Boolean = true,
): String {
val id = request.id
val method = request.method
val json = rpcChannel.encodeRequest(request)
if (BuildConfig.DEBUG) {
Log.d(TAG, formatSafeOutgoingFrameLog(id, method, request.params.keys, json.length, queued = false))
}
var reconnect = false
synchronized(outboundLock) {
if (method == WsMethods.PROMPT_SUBMIT) {
pendingPromptSubmits.add(id)
pendingReply = true
}
val ws = webSocket
if (ws != null && connected.get()) {
// Never replay send(true): the server may have executed it even if its response is lost.
if (ws.send(json)) {
onSent?.invoke(id)
} else {
if (webSocket !== ws || !acceptQueuedMessages.get()) return@synchronized
if (queueIfDisconnected && isRetryableMessage(json)) {
onSent?.invoke(id)
Log.w(TAG, "WS rejected outgoing message — queuing for reconnect")
queueMessage(id, json)
recoverRejectedSocket(ws)
} else {
Log.w(TAG, "WS rejected oversized outgoing message — not retrying")
}
}
} else if (acceptQueuedMessages.get() && queueIfDisconnected) {
if (isRetryableMessage(json)) {
onSent?.invoke(id)
Log.d(TAG, "WS disconnected — queuing message")
queueMessage(id, json)
reconnect = true
} else {
Log.w(TAG, "WS disconnected with oversized outgoing message — not queueing")
}
}
}
if (reconnect) connect()
return id
}
fun removeQueuedMessage(id: String) {
synchronized(outboundLock) {
queuedMessagesById.remove(id)?.let(messageQueue::remove)
}
}
fun onRpcResult(id: String) {
synchronized(outboundLock) {
pendingPromptSubmits.remove(id)
}
}
fun onRpcError(id: String) {
synchronized(outboundLock) {
if (pendingPromptSubmits.remove(id) && pendingPromptSubmits.isEmpty()) {
pendingReply = false
disconnectIfIdleInBackground()
}
}
}
fun observeEvent(event: WsEvent) {
when (event) {
is WsEvent.MessageStart,
is WsEvent.MessageToken,
is WsEvent.ThinkingDelta,
is WsEvent.ReasoningDelta,
is WsEvent.ToolStart,
-> {
pendingReply = true
}
is WsEvent.MessageComplete -> {
pendingReply = false
disconnectIfIdleInBackground()
}
else -> {}
}
}
private fun isRetryableMessage(json: String): Boolean {
if (json.length > MAX_OUTBOUND_MESSAGE_BYTES) return false
if (json.length <= MAX_OUTBOUND_MESSAGE_BYTES / 4) return true
return json.utf8Size() <= MAX_OUTBOUND_MESSAGE_BYTES.toLong()
}
private fun queueMessage(
id: String,
json: String,
) {
if (queuedMessagesById.putIfAbsent(id, json) == null) messageQueue.add(json)
}
private fun markQueuedMessageSent(json: String) {
queuedMessagesById.entries.removeIf { it.value == json }
}
private fun recoverRejectedSocket(ws: WebSocket) {
connected.set(false)
if (closingSocket === ws) return
_connectionStatus.value = ConnectionStatus.RECONNECTING
if (ws.queueSize() == 0L) {
ws.cancel()
scheduleReconnect()
return
}
if (intentionalClose.get()) return
outboundDrainJob?.cancel()
outboundDrainJob =
scope.launch {
delay(OUTBOUND_DRAIN_TIMEOUT_MS)
synchronized(outboundLock) {
if (!connected.get() && !intentionalClose.get() && webSocket === ws) {
ws.cancel()
outboundDrainJob = null
scheduleReconnect()
}
}
}
}
fun sendMessage(
sessionId: String,
text: String,
onSent: ((String) -> Unit)? = null,
queued: Boolean = false,
): String =
send(
rpcChannel.createRequest(
method = RpcMethods.PROMPT_SUBMIT.name,
params =
PromptSubmitParams(
sessionId = sessionId,
text = text,
queued = queued.takeIf { it },
).let { rpcChannel.encodeParams(RpcMethods.PROMPT_SUBMIT, it) },
),
onSent = onSent,
)
fun sendRedirect(
sessionId: String,
text: String,
onSent: ((String) -> Unit)? = null,
): String =
send(
rpcChannel.createRequest(
method = RpcMethods.SESSION_REDIRECT.name,
params =
SessionCorrectionParams(
sessionId = sessionId,
text = text,
).let { rpcChannel.encodeParams(RpcMethods.SESSION_REDIRECT, it) },
),
onSent = onSent,
)
internal fun openSocket() {
val generation =
synchronized(outboundLock) {
if (intentionalClose.get()) return
connectionGeneration.incrementAndGet()
}
scope.launch(Dispatchers.IO) {
if (!refreshWsTicketIfNeeded(generation)) {
Log.w(TAG, "Aborting openSocket: WS ticket refresh failed")
return@launch
}
if (!isCurrentGeneration(generation)) return@launch
val url = AuthManager.wsUrl()
val safeUrl = url.replace(Regex("token=[^&]+"), "token=REDACTED")
if (BuildConfig.DEBUG) Log.d(TAG, "Connecting to $safeUrl")
val request = Request.Builder().url(url).build()
val newSocket = OkHttpProvider.websocket.newWebSocket(request, WsListenerImpl(generation))
synchronized(outboundLock) {
if (connectionGeneration.get() == generation && !intentionalClose.get()) {
webSocket = newSocket
} else {
newSocket.cancel()
}
}
}
}
private fun isCurrentGeneration(generation: Int): Boolean =
connectionGeneration.get() == generation && !intentionalClose.get()
private fun scheduleReconnect() {
synchronized(outboundLock) {
if (intentionalClose.get() ||
_connectionStatus.value == ConnectionStatus.AUTH_EXPIRED ||
reconnectJob?.isActive == true
) {
return
}
if (!AuthManager.isAutoReconnect()) {
if (BuildConfig.DEBUG) Log.d(TAG, "Auto-reconnect disabled")
_connectionStatus.value = ConnectionStatus.DISCONNECTED
return
}
if (!NetworkMonitor.isConnected.value) {
Log.d(TAG, "No network available — delaying reconnect scheduling")
_connectionStatus.value = ConnectionStatus.NO_NETWORK
return
}
val reconnectDelay = reconnectPolicy.nextDelayMs()
reconnectPolicy.advance()
if (BuildConfig.DEBUG) Log.d(TAG, "Reconnecting in ${reconnectDelay}ms …")
reconnectJob =
scope.launch {
delay(reconnectDelay)
val owner = currentCoroutineContext()[Job]
val shouldOpen =
synchronized(outboundLock) {
if (reconnectJob !== owner) {
false
} else {
reconnectJob = null
!intentionalClose.get() &&
!connected.get() &&
_connectionStatus.value != ConnectionStatus.AUTH_EXPIRED
}
}
if (shouldOpen) openSocket()
}
}
}
private fun isTerminalAuthClose(
code: Int,
reason: String,
): Boolean =
code == 4401 || code == 4403 ||
reason.contains("unauthorized", ignoreCase = true) ||
reason.startsWith("auth:", ignoreCase = true)
private inner class WsListenerImpl(
private val generation: Int,
) : WebSocketListener() {
private fun isCurrent(): Boolean = isCurrentGeneration(generation)
override fun onOpen(
webSocket: WebSocket,
response: Response,
) {
synchronized(outboundLock) {
if (!isCurrent()) {
webSocket.close(1000, "Superseded")
return
}
this@WsTransport.webSocket = webSocket
closingSocket = null
outboundDrainJob?.cancel()
outboundDrainJob = null
Log.i(TAG, "WebSocket opened")
connected.set(true)
_connectionStatus.value = ConnectionStatus.CONNECTED
reconnectPolicy.reset()
connectionHealth.start()
while (true) {
val msg = messageQueue.peek() ?: break
if (!isRetryableMessage(msg)) {
Log.w(TAG, "Dropping oversized queued message")
messageQueue.poll()
markQueuedMessageSent(msg)
continue
}
if (BuildConfig.DEBUG) Log.d(TAG, formatSafeQueuedFrameLog(msg))
if (!webSocket.send(msg)) {
recoverRejectedSocket(webSocket)
break
}
messageQueue.poll()
markQueuedMessageSent(msg)
}
replayTracker.triggerReplay()
}
disconnectIfIdleInBackground()
}
override fun onMessage(
webSocket: WebSocket,
text: String,
) {
if (!isCurrent() || this@WsTransport.webSocket !== webSocket) return
if (BuildConfig.DEBUG) Log.d(TAG, formatSafeIncomingFrameLog(text))
connectionHealth.onInboundFrame()
onRawMessage(text)
}
override fun onClosing(
webSocket: WebSocket,
code: Int,
reason: String,
) {
synchronized(outboundLock) {
if (!isCurrent()) {
webSocket.close(code, reason)
return
}
closingSocket = webSocket
// Do NOT log [reason] — it may carry server-side context.
Log.d(TAG, "WebSocket closing: $code")
if (isTerminalAuthClose(code, reason)) {
_connectionStatus.value = ConnectionStatus.AUTH_EXPIRED
}
webSocket.close(code, reason)
}
}
override fun onClosed(
webSocket: WebSocket,
code: Int,
reason: String,
) {
synchronized(outboundLock) {
if (!isCurrent()) return
connectionGeneration.incrementAndGet()
closingSocket = null
outboundDrainJob?.cancel()
outboundDrainJob = null
if (this@WsTransport.webSocket === webSocket) this@WsTransport.webSocket = null
// Do NOT log [reason] — it may carry server-side context. The
// reason is still inspected internally to detect auth failures.
Log.i(TAG, "WebSocket closed: $code")
connected.set(false)
ActiveSessionHolder.clear()
connectionHealth.stop()
rpcChannel.clearCapabilityRequests()
if (isTerminalAuthClose(code, reason)) {
_connectionStatus.value = ConnectionStatus.AUTH_EXPIRED
} else if (_connectionStatus.value != ConnectionStatus.AUTH_EXPIRED) {
_connectionStatus.value = ConnectionStatus.RECONNECTING
scheduleReconnect()
}
}
}
override fun onFailure(
webSocket: WebSocket,
t: Throwable,
response: Response?,
) {
synchronized(outboundLock) {
if (!isCurrent()) return
connectionGeneration.incrementAndGet()
closingSocket = null
outboundDrainJob?.cancel()
outboundDrainJob = null
if (this@WsTransport.webSocket === webSocket) this@WsTransport.webSocket = null
// Log the exception class only — [Throwable.message] can leak URLs
// or headers. The message is still inspected internally for auth
// detection.
Log.e(TAG, "WebSocket failure: ${t.javaClass.simpleName}", t)
connected.set(false)
ActiveSessionHolder.clear()
connectionHealth.stop()
rpcChannel.clearCapabilityRequests()
val code = response?.code ?: 0
if (code == 401 || code == 4401 || code == 4403 ||
t.message?.contains("401") == true ||
t.message?.contains("4401") == true ||
t.message?.contains("4403") == true ||
t.message?.contains("unauthorized", ignoreCase = true) == true
) {
_connectionStatus.value = ConnectionStatus.AUTH_EXPIRED
} else if (_connectionStatus.value != ConnectionStatus.AUTH_EXPIRED) {
_connectionStatus.value = ConnectionStatus.RECONNECTING
scheduleReconnect()
}
}
}
}
// ── Test seams (issue #1376 extraction support) ──────────────────────
@VisibleForTesting
internal val connectedForTest: AtomicBoolean
get() = connected
@VisibleForTesting
internal val intentionalCloseForTest: AtomicBoolean
get() = intentionalClose
@VisibleForTesting
internal val connectionGenerationForTest: AtomicInteger
get() = connectionGeneration
@VisibleForTesting
internal val messageQueueForTest: ConcurrentLinkedQueue<String>
get() = messageQueue
@VisibleForTesting
internal val outboundLockForTest: Any
get() = outboundLock
@VisibleForTesting
internal val acceptQueuedMessagesForTest: AtomicBoolean
get() = acceptQueuedMessages
@VisibleForTesting
internal val connectionStatusForTest: MutableStateFlow<ConnectionStatus>
get() = _connectionStatus
@VisibleForTesting
internal var webSocketForTest: WebSocket?
get() = webSocket
set(value) {
webSocket = value
}
@VisibleForTesting
internal var currentBackoffForTest: Long
get() = synchronized(outboundLock) { reconnectPolicy.currentBackoff }
set(value) {
synchronized(outboundLock) {
reconnectPolicy.currentBackoff = value
}
}
@VisibleForTesting
internal var reconnectJobForTest: Job?
get() = reconnectJob
set(value) {
reconnectJob = value
}
@VisibleForTesting
internal var outboundDrainJobForTest: Job?
get() = outboundDrainJob
set(value) {
outboundDrainJob = value
}
@VisibleForTesting
internal fun setReconnectBackoffForTest(initialMillis: Long) {
synchronized(outboundLock) {
reconnectPolicy.setBackoffForTest(initialMillis)
}
}
@VisibleForTesting
internal fun scheduleReconnectForTest() {
scheduleReconnect()
}
@VisibleForTesting
internal fun createListenerForTest(generation: Int): WebSocketListener = WsListenerImpl(generation)
@VisibleForTesting
internal fun setConnectionStatusForTest(status: ConnectionStatus) {
_connectionStatus.value = status
}
@VisibleForTesting
internal fun formatSafeOutgoingFrameLog(
id: String,
method: String,
paramsKeys: Collection<String>?,
byteLength: Int,
queued: Boolean = false,
): String {
val prefix = if (queued) "→ (queued)" else "→"
val keys = paramsKeys?.joinToString(",", prefix = "[", postfix = "]") ?: "[]"
return "$prefix id=$id method=$method paramsKeys=$keys bytes=$byteLength"
}
@VisibleForTesting
internal fun formatSafeQueuedFrameLog(msg: String): String =
try {
val req = OkHttpProvider.json.decodeFromString<JsonRpcRequest>(msg)
formatSafeOutgoingFrameLog(req.id, req.method, req.params.keys, msg.length, queued = true)
} catch (_: Throwable) {
"→ (queued) frame bytes=${msg.length}"
}
@VisibleForTesting
internal fun formatSafeIncomingFrameLog(text: String): String =
try {
val element = OkHttpProvider.json.parseToJsonElement(text)
if (element is JsonObject) {
val id = (element["id"] as? JsonPrimitive)?.content
val method = (element["method"] as? JsonPrimitive)?.content
val hasResult = element.containsKey("result")
val resultKeys =
(element["result"] as? JsonObject)?.keys?.joinToString(
",",
prefix = "[",
postfix = "]",
)
val errorObj = element["error"] as? JsonObject
val errorCode = (errorObj?.get("code") as? JsonPrimitive)?.content
val sb = StringBuilder("← ")
if (id != null) sb.append("id=").append(id).append(" ")
if (method != null) sb.append("method=").append(method).append(" ")
if (hasResult) sb.append("result=").append(resultKeys ?: "present").append(" ")
if (errorCode != null) sb.append("errorCode=").append(errorCode).append(" ")
sb.append("bytes=").append(text.length).toString()
} else {
"← non-object frame bytes=${text.length}"
}
} catch (_: Throwable) {
"← frame bytes=${text.length}"
}
}
@@ -0,0 +1,518 @@
package com.m57.hermescontrol.data.ws
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.delay
import kotlinx.coroutines.test.advanceTimeBy
import kotlinx.coroutines.test.runCurrent
import kotlinx.coroutines.test.runTest
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Assert.fail
import org.junit.Test
import java.util.concurrent.atomic.AtomicInteger
@OptIn(ExperimentalCoroutinesApi::class)
class ConnectionHealthTest {
private class Fixture(
val scope: CoroutineScope,
var connected: Boolean = true,
var pingDelayMs: Long = 100L,
var pingShouldFail: Boolean = false,
var pendingReplyState: Boolean = false,
var initialTimeMs: Long = 100_000L,
) {
var currentTimeMs: Long = initialTimeMs
val pingCalls = mutableListOf<Long>()
val cancelCalls = AtomicInteger(0)
val logs = mutableListOf<String>()
val debugLogs = mutableListOf<String>()
val health: ConnectionHealth =
ConnectionHealth(
scope = scope,
isConnected = { connected },
pingRequest = { timeoutMs ->
pingCalls.add(timeoutMs)
if (pingShouldFail) {
throw IllegalStateException("Timed out after ${timeoutMs}ms")
}
if (pingDelayMs > 0) {
delay(pingDelayMs)
currentTimeMs += pingDelayMs
}
},
cancelSocket = {
cancelCalls.incrementAndGet()
},
pendingReply = { pendingReplyState },
nowMs = { currentTimeMs },
log = { logs.add(it) },
debugLog = { debugLogs.add(it) },
)
}
@Test
fun pingClockElapsedAndFlowUpdates() =
runTest {
val fixture =
Fixture(
scope = this,
pingDelayMs = 120L,
initialTimeMs = 1_000L,
)
try {
assertEquals(1_000L, fixture.currentTimeMs)
val latency = fixture.health.ping(timeoutMs = 5000L)
assertEquals(120L, latency)
assertEquals(120L, fixture.health.lastLatencyMs.value)
assertEquals(1_120L, fixture.health.lastPongTimestamp)
assertEquals(listOf(5000L), fixture.pingCalls)
} finally {
fixture.health.stop()
}
}
@Test
fun pingFailureLeavesPreviousLatencyUntouched() =
runTest {
val fixture =
Fixture(
scope = this,
pingDelayMs = 50L,
initialTimeMs = 5_000L,
)
try {
val firstLatency = fixture.health.ping(timeoutMs = 5000L)
assertEquals(50L, firstLatency)
assertEquals(50L, fixture.health.lastLatencyMs.value)
fixture.pingShouldFail = true
try {
fixture.health.ping(timeoutMs = 2000L)
fail("Expected ping failure")
} catch (expected: IllegalStateException) {
// Expected
}
// Previous latency state is preserved
assertEquals(50L, fixture.health.lastLatencyMs.value)
} finally {
fixture.health.stop()
}
}
@Test
fun stopResetsLatencyToNull() =
runTest {
val fixture =
Fixture(
scope = this,
pingDelayMs = 40L,
)
try {
fixture.health.ping()
assertEquals(40L, fixture.health.lastLatencyMs.value)
fixture.health.stop()
assertNull(fixture.health.lastLatencyMs.value)
} finally {
fixture.health.stop()
}
}
@Test
fun onInboundFrameUpdatesPongTimestamp() =
runTest {
val fixture =
Fixture(
scope = this,
initialTimeMs = 10_000L,
)
try {
assertEquals(0L, fixture.health.lastPongTimestamp)
fixture.health.onInboundFrame()
assertEquals(10_000L, fixture.health.lastPongTimestamp)
fixture.currentTimeMs = 25_000L
fixture.health.onInboundFrame()
assertEquals(25_000L, fixture.health.lastPongTimestamp)
} finally {
fixture.health.stop()
}
}
@Test
fun isHealthyStrictlyEnforcesStaleThresholdOf30000() =
runTest {
val fixture =
Fixture(
scope = this,
connected = true,
initialTimeMs = 50_000L,
)
try {
fixture.health.onInboundFrame()
assertEquals(50_000L, fixture.health.lastPongTimestamp)
// now - lastPong < 30000 is healthy
fixture.currentTimeMs = 50_000L + 29_999L
assertTrue(fixture.health.isHealthy)
// now - lastPong == 30000 is NOT healthy (< 30000 required)
fixture.currentTimeMs = 50_000L + 30_000L
assertFalse(fixture.health.isHealthy)
// now - lastPong > 30000 is NOT healthy
fixture.currentTimeMs = 50_000L + 30_001L
assertFalse(fixture.health.isHealthy)
} finally {
fixture.health.stop()
}
}
@Test
fun isHealthyDisconnectedGuard() =
runTest {
val fixture =
Fixture(
scope = this,
connected = true,
initialTimeMs = 10_000L,
)
try {
fixture.health.onInboundFrame()
assertTrue(fixture.health.isHealthy)
fixture.connected = false
assertFalse(fixture.health.isHealthy)
} finally {
fixture.health.stop()
}
}
@Test
fun forceHealthCheckForTestWatchdogStrictThresholdCancelsSocket() =
runTest {
val fixture =
Fixture(
scope = this,
connected = true,
initialTimeMs = 100_000L,
)
try {
fixture.health.onInboundFrame()
// Within stale threshold (30000ms), no cancellation
fixture.health.forceHealthCheckForTest(30_000L)
assertEquals(0, fixture.cancelCalls.get())
// Exceeding stale threshold (> 30000ms), cancels socket
fixture.health.forceHealthCheckForTest(30_001L)
assertEquals(1, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun periodicHeartbeatCancelsSocketWhenStaleThresholdExceeded() =
runTest {
val fixture =
Fixture(
scope = this,
connected = true,
pingShouldFail = true,
pingDelayMs = 0L,
initialTimeMs = 10_000L,
)
try {
fixture.health.start()
runCurrent()
fixture.health.onInboundFrame()
// Advance by heartbeat interval (15000ms), still fresh
advanceTimeBy(15_000L)
fixture.currentTimeMs += 15_000L
runCurrent()
assertEquals(0, fixture.cancelCalls.get())
// Advance another heartbeat interval to 30000ms from last inbound frame
advanceTimeBy(15_000L)
fixture.currentTimeMs += 15_000L
runCurrent()
assertEquals(0, fixture.cancelCalls.get())
// Now time elapsed is > 30000ms (e.g. 30001ms)
fixture.currentTimeMs += 1L
advanceTimeBy(15_000L)
fixture.currentTimeMs += 15_000L
runCurrent()
assertEquals(1, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun setLastLatencyForTestSetsFlowDirectly() =
runTest {
val fixture =
Fixture(
scope = this,
)
try {
assertNull(fixture.health.lastLatencyMs.value)
fixture.health.setLastLatencyForTest(75L)
assertEquals(75L, fixture.health.lastLatencyMs.value)
fixture.health.setLastLatencyForTest(null)
assertNull(fixture.health.lastLatencyMs.value)
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnTransportChangeSuccess() =
runTest {
val fixture =
Fixture(
scope = this,
pingDelayMs = 25L,
)
try {
var customActionCalled = false
fixture.health.probeLivenessOnTransportChange(
timeoutMs = 5000L,
onFailureAction = { customActionCalled = true },
)
runCurrent()
assertEquals(listOf(5000L), fixture.pingCalls)
assertFalse(customActionCalled)
assertEquals(0, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnTransportChangeFailureInvokesCustomAction() =
runTest {
val fixture =
Fixture(
scope = this,
pingShouldFail = true,
)
try {
var customActionCalled = false
fixture.health.probeLivenessOnTransportChange(
timeoutMs = 4000L,
onFailureAction = { customActionCalled = true },
)
runCurrent()
assertEquals(listOf(4000L), fixture.pingCalls)
assertTrue(customActionCalled)
assertEquals(0, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnTransportChangeFailureInvokesDefaultCancelSocket() =
runTest {
val fixture =
Fixture(
scope = this,
pingShouldFail = true,
)
try {
fixture.health.probeLivenessOnTransportChange(
timeoutMs = 5000L,
)
runCurrent()
assertEquals(listOf(5000L), fixture.pingCalls)
assertEquals(1, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnWakeIdleCancelsImmediatelyOnFailure() =
runTest {
val fixture =
Fixture(
scope = this,
pingShouldFail = true,
pendingReplyState = false,
)
try {
fixture.health.probeLivenessOnWake(deferredOnce = false)
runCurrent()
assertEquals(listOf(5000L), fixture.pingCalls)
assertEquals(1, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnWakePendingReplyFirstFailureDefers3000MsAndRetriesOnce() =
runTest {
val fixture =
Fixture(
scope = this,
pingShouldFail = true,
pendingReplyState = true,
)
try {
fixture.health.probeLivenessOnWake(deferredOnce = false)
runCurrent()
// First probe fails, but because pendingReply=true, it does not cancel immediately.
// It schedules a deferred retry after 3000ms.
assertEquals(1, fixture.pingCalls.size)
assertEquals(0, fixture.cancelCalls.get())
// Advance before 3000ms
advanceTimeBy(2999L)
runCurrent()
assertEquals(1, fixture.pingCalls.size)
assertEquals(0, fixture.cancelCalls.get())
// Advance to 3000ms - second ping is executed
advanceTimeBy(1L)
runCurrent()
assertEquals(2, fixture.pingCalls.size)
// Second probe fails, now it cancels
assertEquals(1, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnWakePendingReplySecondProbeSucceedsLeavesSocketOpen() =
runTest {
val fixture =
Fixture(
scope = this,
pingShouldFail = true,
pendingReplyState = true,
pingDelayMs = 10L,
)
try {
fixture.health.probeLivenessOnWake(deferredOnce = false)
runCurrent()
// First attempt failed, pending deferred retry in 3000ms
assertEquals(1, fixture.pingCalls.size)
assertEquals(0, fixture.cancelCalls.get())
// Before retry fires, recovery occurs: pings now succeed
fixture.pingShouldFail = false
advanceTimeBy(3000L)
runCurrent()
assertEquals(2, fixture.pingCalls.size)
assertEquals(0, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun cancelForegroundProbeCancelsPendingProbe() =
runTest {
val fixture =
Fixture(
scope = this,
pingShouldFail = true,
pendingReplyState = true,
)
try {
fixture.health.probeLivenessOnWake(deferredOnce = false)
runCurrent()
assertEquals(1, fixture.pingCalls.size)
assertEquals(0, fixture.cancelCalls.get())
// Cancel the scheduled retry
fixture.health.cancelForegroundProbe()
advanceTimeBy(5000L)
runCurrent()
// No second probe should have been made, no cancellation
assertEquals(1, fixture.pingCalls.size)
assertEquals(0, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnTransportChangeSuccessLogsAtDebugLevelWithoutWarning() =
runTest {
val fixture =
Fixture(
scope = this,
pingDelayMs = 25L,
)
try {
fixture.health.probeLivenessOnTransportChange(timeoutMs = 5000L)
runCurrent()
advanceTimeBy(25L)
runCurrent()
assertEquals(listOf(5000L), fixture.pingCalls)
assertEquals(
listOf(
"Network transport changed — probing WebSocket liveness",
"WebSocket liveness probe succeeded on new transport",
),
fixture.debugLogs,
)
assertTrue(fixture.logs.isEmpty())
assertEquals(0, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
@Test
fun probeLivenessOnTransportChangeFailureLogsDebugAndWarningAndCancelsSocket() =
runTest {
val fixture =
Fixture(
scope = this,
pingShouldFail = true,
)
try {
fixture.health.probeLivenessOnTransportChange(timeoutMs = 5000L)
runCurrent()
assertEquals(listOf(5000L), fixture.pingCalls)
assertEquals(
listOf("Network transport changed — probing WebSocket liveness"),
fixture.debugLogs,
)
assertEquals(
listOf("Transport change liveness probe failed — cancelling socket"),
fixture.logs,
)
assertEquals(1, fixture.cancelCalls.get())
} finally {
fixture.health.stop()
}
}
}
@@ -0,0 +1,532 @@
package com.m57.hermescontrol.data.ws
import android.util.Log
import com.m57.hermescontrol.data.local.AuthManager
import com.m57.hermescontrol.data.remote.CookieManager
import com.m57.hermescontrol.data.remote.DashboardSessionTokenRefresher
import com.m57.hermescontrol.data.remote.OkHttpProvider
import com.m57.hermescontrol.data.remote.buildFakePersistentCookieJar
import io.mockk.coEvery
import io.mockk.every
import io.mockk.mockk
import io.mockk.mockkObject
import io.mockk.mockkStatic
import io.mockk.unmockkAll
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.onSubscription
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import okhttp3.WebSocket
import okhttp3.WebSocketListener
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import org.junit.After
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Assert.fail
import org.junit.Before
import org.junit.Test
import java.util.Collections
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicReference
/**
* Characterization test suite capturing existing baseline behaviors and edge cases
* of [HermesWsClient] before extracting components in issue #1376.
*/
class HermesWsClientCharacterizationTest {
private lateinit var mockWebServer: MockWebServer
private val serverSockets = Collections.synchronizedList(mutableListOf<WebSocket>())
private lateinit var serverClosed: CountDownLatch
@Before
fun setUp() {
serverSockets.clear()
serverClosed = CountDownLatch(1)
mockkStatic(Log::class)
every { Log.isLoggable(any(), any()) } returns false
every { Log.d(any<String>(), any<String>()) } returns 0
every { Log.i(any<String>(), any<String>()) } returns 0
every { Log.w(any<String>(), any<String>()) } returns 0
every { Log.w(any<String>(), any<String>(), any<Throwable>()) } returns 0
every { Log.e(any<String>(), any<String>()) } returns 0
every { Log.e(any<String>(), any<String>(), any()) } returns 0
mockWebServer = MockWebServer()
mockWebServer.start()
mockkObject(AuthManager)
every { AuthManager.wsUrl() } returns mockWebServer.url("/").toString().replace("http://", "ws://")
every { AuthManager.isAutoReconnect() } returns false
every { AuthManager.getSessionCookie() } returns null
every { AuthManager.serverStore } returns
mockk<com.m57.hermescontrol.data.config.ServerStore>().also {
every { it.getLatestState() } returns
com.m57.hermescontrol.data.config
.ServerStoreState()
}
CookieManager.setJarForTest(buildFakePersistentCookieJar())
mockkObject(DashboardSessionTokenRefresher)
coEvery { DashboardSessionTokenRefresher.refreshAsync() } returns null
every { DashboardSessionTokenRefresher.refresh() } returns null
HermesWsClient.connectedForTest.set(false)
HermesWsClient.connectionStatusForTest.value = ConnectionStatus.DISCONNECTED
HermesWsClient.messageQueueForTest.clear()
HermesWsClient.disconnect(clearPendingMessages = true)
HermesWsClient.releaseExternalActivityConnectionLease()
HermesWsClient.releaseBackgroundConnectionLease()
HermesWsClient.setAppForeground(true)
HermesWsClient.acceptQueuedMessagesForTest.set(true)
HermesWsClient.clearCredentialWarning()
HermesWsClient.clearSeqWatermarks()
HermesWsClient.setLastLatencyForTest(null)
}
@After
fun tearDown() {
HermesWsClient.rejectAllPending()
HermesWsClient.releaseExternalActivityConnectionLease()
HermesWsClient.releaseBackgroundConnectionLease()
HermesWsClient.disconnect(clearPendingMessages = true)
runBlocking {
withTimeout(3000) {
HermesWsClient.connectionStatus.first { it == ConnectionStatus.DISCONNECTED }
}
}
try {
if (serverSockets.isNotEmpty()) {
assertTrue("Server endpoint must finish closing", serverClosed.await(3, TimeUnit.SECONDS))
}
mockWebServer.shutdown()
} finally {
unmockkAll()
}
}
private fun connectClientToServer(onServerOpen: ((WebSocket) -> Unit)? = null): WebSocket {
val serverOpenLatch = CountDownLatch(1)
var serverWs: WebSocket? = null
mockWebServer.enqueue(
MockResponse().withWebSocketUpgrade(
object : WebSocketListener() {
override fun onClosing(
webSocket: WebSocket,
code: Int,
reason: String,
) {
webSocket.close(code, reason)
}
override fun onClosed(
webSocket: WebSocket,
code: Int,
reason: String,
) {
serverClosed.countDown()
}
override fun onFailure(
webSocket: WebSocket,
t: Throwable,
response: okhttp3.Response?,
) {
serverClosed.countDown()
}
override fun onOpen(
webSocket: WebSocket,
response: okhttp3.Response,
) {
serverSockets.add(webSocket)
serverWs = webSocket
onServerOpen?.invoke(webSocket)
serverOpenLatch.countDown()
}
},
),
)
HermesWsClient.connect()
runBlocking {
withTimeout(5000) {
HermesWsClient.connectionStatus.first { it == ConnectionStatus.CONNECTED }
}
}
assertTrue("Server WebSocket should open", serverOpenLatch.await(5, TimeUnit.SECONDS))
return requireNotNull(serverWs)
}
@Test
fun credentialWarning_updatedByGatewayReadyAndSessionInfo_retainedOnBlank_clearedExplicitly() {
assertNull(HermesWsClient.credentialWarning.value)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectedForTest.set(true)
val dummySocket =
mockk<WebSocket>(relaxed = true) {
every { send(any<String>()) } returns true
}
HermesWsClient.webSocketForTest = dummySocket
val generation = HermesWsClient.connectionGenerationForTest.get()
val listener = HermesWsClient.createListenerForTest(generation)
// 1. GatewayReady with non-blank credential_warning updates flow
val gatewayReadyFrame =
"""{"jsonrpc":"2.0","method":"event","params":{"type":"gateway.ready","payload":{"credential_warning":"Warning: Auth key expired"}}}"""
listener.onMessage(dummySocket, gatewayReadyFrame)
runBlocking {
withTimeout(3000) {
HermesWsClient.credentialWarning.first { it == "Warning: Auth key expired" }
}
}
assertEquals("Warning: Auth key expired", HermesWsClient.credentialWarning.value)
// 2. Blank or absent credential_warning does NOT clear the existing value.
// Deliver follow-up gateway.ready (absent) after blank and check warning is retained,
// then use session.info barrier to prove processing pipeline is drained.
val blankFrame =
"""{"jsonrpc":"2.0","method":"event","params":{"type":"gateway.ready","payload":{"credential_warning":""}}}"""
listener.onMessage(dummySocket, blankFrame)
assertEquals("Warning: Auth key expired", HermesWsClient.credentialWarning.value)
val absentFrame =
"""{"jsonrpc":"2.0","method":"event","params":{"type":"gateway.ready","payload":{"version":"1.0"}}}"""
listener.onMessage(dummySocket, absentFrame)
assertEquals("Warning: Auth key expired", HermesWsClient.credentialWarning.value)
// Barrier: a distinct non-blank session.info proves all preceding frames were processed in order
val barrierFrame =
"""{"jsonrpc":"2.0","method":"event","params":{"type":"session.info","session_id":"s-barrier","payload":{"credential_warning":"Barrier reached"}}}"""
listener.onMessage(dummySocket, barrierFrame)
runBlocking {
withTimeout(3000) {
HermesWsClient.credentialWarning.first { it == "Barrier reached" }
}
}
assertEquals("Barrier reached", HermesWsClient.credentialWarning.value)
// 3. SessionInfo with non-blank updates warning
val sessionInfoFrame =
"""{"jsonrpc":"2.0","method":"event","params":{"type":"session.info","session_id":"s1","payload":{"credential_warning":"Session warning"}}}"""
listener.onMessage(dummySocket, sessionInfoFrame)
runBlocking {
withTimeout(3000) {
HermesWsClient.credentialWarning.first { it == "Session warning" }
}
}
assertEquals("Session warning", HermesWsClient.credentialWarning.value)
// 4. Explicit clear resets to null
HermesWsClient.clearCredentialWarning()
assertNull(HermesWsClient.credentialWarning.value)
}
@Test
fun disconnect_resetsLastLatencyMsToNull() {
HermesWsClient.setLastLatencyForTest(42L)
assertEquals(42L, HermesWsClient.lastLatencyMs.value)
HermesWsClient.disconnect()
assertNull(HermesWsClient.lastLatencyMs.value)
}
@Test
fun rpcTimeout_emitsHermesRpcExceptionCodeMinusOneExactMessage() {
val serverWs = connectClientToServer()
assertNotNull(serverWs)
val shortTimeoutMs = 150L
val deferred = HermesWsClient.request("custom.test_method", emptyMap(), timeoutMs = shortTimeoutMs)
try {
runBlocking {
withTimeout(3000) {
deferred.await()
}
}
fail("Expected deferred to fail with timeout exception")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals(-1, e.code)
assertEquals("Request timed out: custom.test_method", e.message)
}
}
@Test
fun rejectAllPending_failsRequestsWithCustomException_cancelsTimeout_clearsPending() {
connectClientToServer()
val deferred = HermesWsClient.request("custom.hold", emptyMap(), timeoutMs = 10_000L)
val pendingIds = HermesWsClient.pendingCallIdsForTest()
assertEquals(1, pendingIds.size)
val reqId = pendingIds.first()
val timeoutJob = HermesWsClient.pendingCallTimeoutForTest(reqId)
assertNotNull("Timeout job should be scheduled", timeoutJob)
assertTrue("Timeout job should be active", timeoutJob?.isActive == true)
val customException = HermesWsClient.HermesRpcException("Custom forced rejection", code = -99)
HermesWsClient.rejectAllPending(customException)
assertTrue(
"Pending map should be empty after rejectAllPending",
HermesWsClient.pendingCallIdsForTest().isEmpty(),
)
assertTrue("Timeout job should be cancelled", timeoutJob?.isCancelled == true)
assertTrue("Deferred should be completed exceptionally", deferred.isCompleted)
try {
runBlocking { deferred.await() }
fail("Expected custom exception")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals(-99, e.code)
assertEquals("Custom forced rejection", e.message)
}
}
@Test
fun reconnectForNetworkChange_rejectsPendingCallsImmediately() {
connectClientToServer()
val deferred = HermesWsClient.request("custom.in_flight", emptyMap(), timeoutMs = 30_000L)
assertEquals(1, HermesWsClient.pendingCallIdsForTest().size)
// Network change triggers immediate rejectAllPending
HermesWsClient.reconnectForNetworkChange(networkAvailable = false)
assertTrue("Pending map should be empty after network change", HermesWsClient.pendingCallIdsForTest().isEmpty())
assertTrue("Deferred must be completed", deferred.isCompleted)
try {
runBlocking { deferred.await() }
fail("Expected pending call to be rejected on network change")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals("Connection lost — request cancelled", e.message)
}
}
@Test
fun disconnectAndListenerClosed_doNotRejectAllPending_deferredRemainsPendingUntilTimeoutOrReject() {
val serverWs = connectClientToServer()
val shortTimeoutMs = 300L
val deferred = HermesWsClient.request("custom.deferred_call", emptyMap(), timeoutMs = shortTimeoutMs)
val reqId = HermesWsClient.pendingCallIdsForTest().first()
// Drive onClosed on the CURRENT active generation before disconnect.
// In onClosed: isCurrent() is true (intentionalClose is still false, generation matches).
// This exercises the active listener onClosed branch.
val activeGeneration = HermesWsClient.connectionGenerationForTest.get()
val currentListener = HermesWsClient.createListenerForTest(activeGeneration)
val activeClientSocket = HermesWsClient.webSocketForTest ?: serverWs
currentListener.onClosed(activeClientSocket, 1000, "Normal closure")
activeClientSocket.cancel()
// Deferred remains pending after active onClosed
assertFalse("Deferred should still remain pending after onClosed", deferred.isCompleted)
assertTrue("Request ID should still be in pendingCalls", HermesWsClient.pendingCallIdsForTest().contains(reqId))
// Calling disconnect() does NOT invoke rejectAllPending() either
HermesWsClient.disconnect()
assertFalse("Deferred should NOT be completed immediately by disconnect() alone", deferred.isCompleted)
assertTrue("Request ID should still be in pendingCalls", HermesWsClient.pendingCallIdsForTest().contains(reqId))
// Eventually, the per-request timeout triggers and resolves it
try {
runBlocking {
withTimeout(2000) {
deferred.await()
}
}
fail("Deferred should fail with timeout")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals(-1, e.code)
assertEquals("Request timed out: custom.deferred_call", e.message)
}
}
@Test
fun combinedExternalAndBackgroundLeases_releaseOneKeepsOther_releaseBothDisconnectsWhenBackgrounded() {
connectClientToServer()
assertTrue(HermesWsClient.isConnected)
// Acquire both leases
HermesWsClient.acquireExternalActivityConnectionLease()
HermesWsClient.acquireBackgroundConnectionLease()
assertTrue(HermesWsClient.hasBackgroundConnectionLease())
// Move to background - should NOT disconnect because leases are active
HermesWsClient.setAppForeground(false)
assertTrue("Should stay connected while leases are active", HermesWsClient.isConnected)
// Release external lease - background lease still holds connection open
HermesWsClient.releaseExternalActivityConnectionLease()
assertTrue("Should stay connected with background lease active", HermesWsClient.isConnected)
// Release background lease - both leases released, app backgrounded, idle -> disconnects
HermesWsClient.releaseBackgroundConnectionLease()
assertFalse("Should disconnect when all leases released in background", HermesWsClient.isConnected)
}
@Test
fun delayedReplayAndLiveRace_incomingSeqBufferedBeforeReplayResponseAndDeduped() {
val sessionId = "session-characterization-race"
val serverOpenLatch = CountDownLatch(1)
val replayReceivedLatch = CountDownLatch(1)
val allowReplayResponseLatch = CountDownLatch(1)
val capturedReplayId = AtomicReference<String>()
var serverSocketRef: WebSocket? = null
// Enqueue MockResponse with WebSocketListener on MockWebServer
mockWebServer.enqueue(
MockResponse().withWebSocketUpgrade(
object : WebSocketListener() {
override fun onClosing(
webSocket: WebSocket,
code: Int,
reason: String,
) {
webSocket.close(code, reason)
}
override fun onClosed(
webSocket: WebSocket,
code: Int,
reason: String,
) {
serverClosed.countDown()
}
override fun onFailure(
webSocket: WebSocket,
t: Throwable,
response: okhttp3.Response?,
) {
serverClosed.countDown()
}
override fun onOpen(
webSocket: WebSocket,
response: okhttp3.Response,
) {
serverSockets.add(webSocket)
serverSocketRef = webSocket
serverOpenLatch.countDown()
}
override fun onMessage(
webSocket: WebSocket,
text: String,
) {
if (text.contains(WsMethods.SESSION_EVENTS_SINCE)) {
val request = OkHttpProvider.json.parseToJsonElement(text) as JsonObject
val id = (request["id"] as JsonPrimitive).content
capturedReplayId.set(id)
replayReceivedLatch.countDown()
if (allowReplayResponseLatch.await(5, TimeUnit.SECONDS)) {
val rpcId = requireNotNull(capturedReplayId.get())
val idJson = JsonPrimitive(rpcId).toString()
val replayResponse =
"""{"jsonrpc":"2.0","id":$idJson,"result":{"epoch":"epoch-1","latest_seq":12,"events":[{"type":"message.complete","session_id":"$sessionId","seq":11,"payload":{"text":"Replayed message 11"}},{"type":"message.complete","session_id":"$sessionId","seq":12,"payload":{"text":"Replayed message 12"}}]}}"""
webSocket.send(replayResponse)
}
}
}
},
),
)
// Set watermark at seq 10 and epoch-1 prior to connect
HermesWsClient.setSeqWatermark(sessionId, 10)
HermesWsClient.setReplayEpochForTest("epoch-1")
// Prepare subscription with onSubscription latch before stimulus
val collectedEvents = Collections.synchronizedList(mutableListOf<WsEvent>())
val subscriptionLatch = CountDownLatch(1)
val collectLatch = CountDownLatch(2)
val liveProcessedLatch = CountDownLatch(1)
val collectorJob =
kotlinx.coroutines.CoroutineScope(kotlinx.coroutines.Dispatchers.IO).launch {
HermesWsClient.events
.onSubscription {
subscriptionLatch.countDown()
}.collect { event ->
if (event is WsEvent.SessionInfo && event.data?.get("barrier") == "live-processed") {
liveProcessedLatch.countDown()
}
if (event is WsEvent.MessageComplete && event.sessionId == sessionId) {
collectedEvents.add(event)
collectLatch.countDown()
}
}
}
try {
assertTrue("Collector subscription should be established", subscriptionLatch.await(3, TimeUnit.SECONDS))
// Connect client - real onOpen automatically arms replay
HermesWsClient.connect()
runBlocking {
withTimeout(5000) {
HermesWsClient.connectionStatus.first { it == ConnectionStatus.CONNECTED }
}
}
assertTrue("Server WebSocket should open", serverOpenLatch.await(5, TimeUnit.SECONDS))
val serverSocket = requireNotNull(serverSocketRef)
// Wait until the server receives session.events.since request
assertTrue("Server should receive replay request", replayReceivedLatch.await(5, TimeUnit.SECONDS))
assertTrue("Replay should be in flight", HermesWsClient.isReplayInFlight())
// 1. Live frame arrived while replay is in flight (seq 12)
val liveFrame =
"""{"jsonrpc":"2.0","method":"event","params":{"type":"message.complete","session_id":"$sessionId","seq":12,"payload":{"text":"Live message 12"}}}"""
serverSocket.send(liveFrame)
// A following unsequenced frame proves the live notification reached the client
// before the replay response is released; unchanged watermark alone is not a barrier.
serverSocket.send(
"""{"method":"event","params":{"type":"session.info","session_id":"$sessionId","payload":{"barrier":"live-processed"}}}""",
)
assertTrue(
"Live frame must be processed before replay completes",
liveProcessedLatch.await(3, TimeUnit.SECONDS),
)
assertEquals(10, HermesWsClient.getSeqWatermarks()[sessionId])
assertEquals("Live frame should be held in replayHold while replay is in flight", 0, collectedEvents.size)
// 2. Unblock replay response from server
allowReplayResponseLatch.countDown()
// Wait for replay to deliver replayed events and drain held live events
assertTrue("Should collect replayed and unheld events", collectLatch.await(5, TimeUnit.SECONDS))
// Watermark should advance to 12
assertEquals(12, HermesWsClient.getSeqWatermarks()[sessionId])
// Deduplication: held live frame seq 12 deduped against replay seq 12
assertEquals(2, collectedEvents.size)
assertEquals("Replayed message 11", (collectedEvents[0] as WsEvent.MessageComplete).text)
assertEquals("Replayed message 12", (collectedEvents[1] as WsEvent.MessageComplete).text)
} finally {
allowReplayResponseLatch.countDown()
collectorJob.cancel()
}
}
}
@@ -11,7 +11,6 @@ import io.mockk.mockkStatic
import io.mockk.unmockkAll
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.onSubscription
import kotlinx.coroutines.launch
@@ -30,11 +29,8 @@ import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import java.util.Collections
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
class HermesWsClientLoggingSecurityTest {
private val capturedDebugLogs = Collections.synchronizedList(mutableListOf<String>())
@@ -87,17 +83,9 @@ class HermesWsClientLoggingSecurityTest {
CookieManager.setJarForTest(buildFakePersistentCookieJar())
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as AtomicBoolean).set(false)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as AtomicBoolean).set(false)
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as AtomicInteger).set(1)
HermesWsClient.connectedForTest.set(false)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionGenerationForTest.set(1)
HermesWsClient.rejectAllPending()
HermesWsClient.disconnect(clearPendingMessages = true)
@@ -113,51 +101,20 @@ class HermesWsClientLoggingSecurityTest {
unmockkAll()
}
private fun createWsListener(generation: Int? = null): WebSocketListener {
val targetGen =
generation ?: run {
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as AtomicInteger).get()
}
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
return constructor.newInstance(targetGen) as WebSocketListener
}
private fun createWsListener(generation: Int? = null): WebSocketListener =
HermesWsClient.createListenerForTest(generation ?: HermesWsClient.connectionGenerationForTest.get())
private fun attachMockSocket(): WebSocket {
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send(any<String>()) } returns true
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, socket)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as AtomicBoolean).set(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as AtomicBoolean).set(false)
HermesWsClient.webSocketForTest = socket
HermesWsClient.connectedForTest.set(true)
HermesWsClient.intentionalCloseForTest.set(false)
return socket
}
@Suppress("UNCHECKED_CAST")
private fun getPendingCalls(): ConcurrentHashMap<String, Any> {
val field = HermesWsClient::class.java.getDeclaredField("pendingCalls")
field.isAccessible = true
return field.get(HermesWsClient) as ConcurrentHashMap<String, Any>
}
private fun getTimeoutJob(pendingCall: Any): Job? {
val field = pendingCall.javaClass.getDeclaredField("timeoutJob")
field.isAccessible = true
return field.get(pendingCall) as? Job
}
// ── Formatter unit tests ──────────────────────────────────────────────────
@Test
@@ -255,8 +212,7 @@ class HermesWsClientLoggingSecurityTest {
val listener = createWsListener()
val deferred = HermesWsClient.request("connectors.connect", mapOf("connector" to "synthetic_provider"))
val pendingCalls = getPendingCalls()
val reqId = pendingCalls.keys.firstOrNull()
val reqId = HermesWsClient.pendingCallIdsForTest().firstOrNull()
assertNotNull("Pending call ID must exist", reqId)
val sensitiveAuthUrl =
@@ -332,8 +288,7 @@ class HermesWsClientLoggingSecurityTest {
assertTrue("Event collector must subscribe", subscribedLatch.await(3, TimeUnit.SECONDS))
val deferred = HermesWsClient.request("session.steer", mapOf("session_id" to "sess-ownership-4001"))
val pendingCalls = getPendingCalls()
val reqId = pendingCalls.keys.firstOrNull()
val reqId = HermesWsClient.pendingCallIdsForTest().firstOrNull()
assertNotNull("Pending call ID must exist", reqId)
val errorJson =
@@ -434,15 +389,9 @@ class HermesWsClientLoggingSecurityTest {
fun `cancelling pending deferred cleans pending requests and cancels timeout job`() {
attachMockSocket()
val deferred = HermesWsClient.request("test.cancellation.method", mapOf("synthetic" to "fixture"))
val pendingCalls = getPendingCalls()
val entry =
pendingCalls.entries.firstOrNull {
it.value.javaClass.name
.contains("PendingCall")
}
assertNotNull("Pending call must be registered in pendingCalls map", entry)
val reqId = entry!!.key
val timeoutJob = getTimeoutJob(entry.value)
val reqId = HermesWsClient.pendingCallIdsForTest().firstOrNull()
assertNotNull("Pending call must be registered in pendingCalls map", reqId)
val timeoutJob = HermesWsClient.pendingCallTimeoutForTest(reqId!!)
assertNotNull("Timeout job must be initialized", timeoutJob)
assertTrue("Timeout job must be active while request is pending", timeoutJob!!.isActive)
@@ -451,7 +400,7 @@ class HermesWsClientLoggingSecurityTest {
assertTrue("Deferred must be cancelled", deferred.isCancelled)
assertFalse(
"Pending call must be removed from pendingCalls map on cancellation",
pendingCalls.containsKey(reqId),
HermesWsClient.pendingCallIdsForTest().contains(reqId),
)
assertTrue("Timeout job must be cancelled on deferred cancellation", timeoutJob.isCancelled)
}
@@ -91,27 +91,15 @@ class HermesWsClientTest {
every { DashboardSessionTokenRefresher.refresh() } returns null
// Reset state
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus")
statusField.isAccessible = true
@Suppress("UNCHECKED_CAST")
(statusField.get(HermesWsClient) as MutableStateFlow<ConnectionStatus>).value = ConnectionStatus.DISCONNECTED
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
(queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>).clear()
HermesWsClient.connectedForTest.set(false)
HermesWsClient.connectionStatusForTest.value = ConnectionStatus.DISCONNECTED
HermesWsClient.messageQueueForTest.clear()
HermesWsClient.disconnect(clearPendingMessages = true) // Ensure it starts clean
HermesWsClient.releaseExternalActivityConnectionLease()
HermesWsClient.releaseBackgroundConnectionLease()
HermesWsClient.setAppForeground(true)
val acceptQueuedMessagesField = HermesWsClient::class.java.getDeclaredField("acceptQueuedMessages")
acceptQueuedMessagesField.isAccessible = true
(acceptQueuedMessagesField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.acceptQueuedMessagesForTest.set(true)
}
@After
@@ -350,10 +338,8 @@ class HermesWsClientTest {
val socket = mockk<WebSocket>()
every { socket.send(any<String>()) } returns true
every { socket.close(any(), any()) } returns true
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket").apply { isAccessible = true }
val connectedField = HermesWsClient::class.java.getDeclaredField("connected").apply { isAccessible = true }
socketField.set(HermesWsClient, socket)
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.webSocketForTest = socket
HermesWsClient.connectedForTest.set(true)
val sent =
HermesWsClient.respondToServerRequest(
@@ -384,10 +370,8 @@ class HermesWsClientTest {
val socket = mockk<WebSocket>()
every { socket.send(any<String>()) } returns true
every { socket.close(any(), any()) } returns true
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket").apply { isAccessible = true }
val connectedField = HermesWsClient::class.java.getDeclaredField("connected").apply { isAccessible = true }
socketField.set(HermesWsClient, socket)
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.webSocketForTest = socket
HermesWsClient.connectedForTest.set(true)
assertTrue(HermesWsClient.respondToServerRequestError("srq-abc123", -32601, "unsupported"))
verify {
@@ -416,29 +400,14 @@ class HermesWsClientTest {
val staleSocket = mockk<WebSocket>(relaxed = true)
every { staleSocket.send(any<String>()) } returns false
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, staleSocket)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus")
statusField.isAccessible = true
@Suppress("UNCHECKED_CAST")
(statusField.get(HermesWsClient) as MutableStateFlow<ConnectionStatus>).value = ConnectionStatus.CONNECTED
HermesWsClient.webSocketForTest = staleSocket
HermesWsClient.connectedForTest.set(true)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionStatusForTest.value = ConnectionStatus.CONNECTED
HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello"))
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
assertFalse(HermesWsClient.isConnected)
assertEquals(ConnectionStatus.DISCONNECTED, HermesWsClient.connectionStatus.value)
assertEquals(1, queue.size)
@@ -446,8 +415,7 @@ class HermesWsClientTest {
@Test
fun testSendRegistersRequestIdWhileOutboundLockHeld() {
val lockField = HermesWsClient::class.java.getDeclaredField("outboundLock").apply { isAccessible = true }
val outboundLock = lockField.get(HermesWsClient)
val outboundLock = HermesWsClient.outboundLockForTest
var callbackHeldLock = false
HermesWsClient.send("test.method", onSent = { callbackHeldLock = Thread.holdsLock(outboundLock) })
@@ -458,25 +426,13 @@ class HermesWsClientTest {
@Test
fun testNetworkChangeInvalidatesOldSocketAndOpensReplacementImmediately() {
every { AuthManager.isAutoReconnect() } returns true
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectedForTest.set(true)
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send(any<String>()) } returns true
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, socket)
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
val generation = generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger
val oldGeneration = generation.get()
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val staleListener = constructor.newInstance(oldGeneration) as WebSocketListener
HermesWsClient.webSocketForTest = socket
val oldGeneration = HermesWsClient.connectionGenerationForTest.get()
val staleListener = HermesWsClient.createListenerForTest(oldGeneration)
var replacementOpens = 0
val deferred = HermesWsClient.request(WsMethods.PROCESS_LIST, mapOf("session_id" to "s1"))
@@ -486,10 +442,10 @@ class HermesWsClientTest {
verify(exactly = 1) { socket.cancel() }
assertEquals(1, replacementOpens)
assertEquals(oldGeneration + 1, generation.get())
assertEquals(oldGeneration + 1, HermesWsClient.connectionGenerationForTest.get())
assertFalse(HermesWsClient.isConnected)
assertEquals(ConnectionStatus.RECONNECTING, HermesWsClient.connectionStatus.value)
assertEquals(null, socketField.get(HermesWsClient))
assertEquals(null, HermesWsClient.webSocketForTest)
assertTrue(deferred.isCompleted)
}
@@ -497,18 +453,10 @@ class HermesWsClientTest {
fun testNetworkLossPreservesAuthExpiredStatus() =
runBlocking {
every { AuthManager.isAutoReconnect() } returns true
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus")
statusField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val status = statusField.get(HermesWsClient) as MutableStateFlow<ConnectionStatus>
status.value = ConnectionStatus.AUTH_EXPIRED
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED
val socket = mockk<WebSocket>(relaxed = true)
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, socket)
HermesWsClient.webSocketForTest = socket
HermesWsClient.reconnectForNetworkChange(false)
HermesWsClient.reconnectForNetworkChange(true)
@@ -520,22 +468,13 @@ class HermesWsClientTest {
@Test
fun testBreakBeforeMakeDoesNotReplayAcceptedRequest() {
every { AuthManager.isAutoReconnect() } returns true
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectedForTest.set(true)
val oldSocket = mockk<WebSocket>(relaxed = true)
every { oldSocket.send(any<String>()) } returns true
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, oldSocket)
HermesWsClient.webSocketForTest = oldSocket
HermesWsClient.send(WsMethods.SUBSCRIPTION_CHANGE, mapOf("cancel" to true))
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
assertTrue(queue.isEmpty())
HermesWsClient.reconnectForNetworkChange(false)
@@ -545,13 +484,7 @@ class HermesWsClientTest {
HermesWsClient.reconnectForNetworkChange(true) { replacementOpens++ }
val replacementSocket = mockk<WebSocket>(relaxed = true)
every { replacementSocket.send(any<String>()) } returns true
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
val generation = generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val replacementListener = constructor.newInstance(generation.get()) as WebSocketListener
val replacementListener = HermesWsClient.createListenerForTest(HermesWsClient.connectionGenerationForTest.get())
replacementListener.onOpen(replacementSocket, mockk(relaxed = true))
assertEquals(1, replacementOpens)
@@ -563,18 +496,10 @@ class HermesWsClientTest {
@Test
fun testNetworkChangePreservesAuthExpiredSocket() {
every { AuthManager.isAutoReconnect() } returns true
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus")
statusField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val status = statusField.get(HermesWsClient) as MutableStateFlow<ConnectionStatus>
status.value = ConnectionStatus.AUTH_EXPIRED
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED
val socket = mockk<WebSocket>(relaxed = true)
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, socket)
HermesWsClient.webSocketForTest = socket
HermesWsClient.reconnectForNetworkChange()
@@ -588,20 +513,13 @@ class HermesWsClientTest {
every { AuthManager.isAutoReconnect() } returns true
every { NetworkMonitor.isConnected } returns MutableStateFlow(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.currentBackoffForTest = 1_000L
val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff")
backoffField.isAccessible = true
backoffField.setLong(HermesWsClient, 1_000L)
HermesWsClient.scheduleReconnectForTest()
HermesWsClient.scheduleReconnectForTest()
val reconnectMethod = HermesWsClient::class.java.getDeclaredMethod("scheduleReconnect")
reconnectMethod.isAccessible = true
reconnectMethod.invoke(HermesWsClient)
reconnectMethod.invoke(HermesWsClient)
assertEquals(2_000L, backoffField.getLong(HermesWsClient))
assertEquals(2_000L, HermesWsClient.currentBackoffForTest)
}
@Test
@@ -610,35 +528,21 @@ class HermesWsClientTest {
every { AuthManager.isAutoReconnect() } returns true
every { NetworkMonitor.isConnected } returns MutableStateFlow(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectedForTest.set(false)
HermesWsClient.currentBackoffForTest = 0L
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff")
backoffField.isAccessible = true
backoffField.setLong(HermesWsClient, 0L)
val reconnectJobField = HermesWsClient::class.java.getDeclaredField("reconnectJob")
reconnectJobField.isAccessible = true
val replacementJob = mockk<Job>(relaxed = true)
val lockField = HermesWsClient::class.java.getDeclaredField("outboundLock")
lockField.isAccessible = true
val lock = lockField.get(HermesWsClient)
val reconnectMethod = HermesWsClient::class.java.getDeclaredMethod("scheduleReconnect")
reconnectMethod.isAccessible = true
val lock = HermesWsClient.outboundLockForTest
synchronized(lock) {
reconnectMethod.invoke(HermesWsClient)
reconnectJobField.set(HermesWsClient, replacementJob)
HermesWsClient.scheduleReconnectForTest()
HermesWsClient.reconnectJobForTest = replacementJob
}
Thread.sleep(100)
assertSame(replacementJob, reconnectJobField.get(HermesWsClient))
assertSame(replacementJob, HermesWsClient.reconnectJobForTest)
}
@Test
@@ -646,23 +550,15 @@ class HermesWsClientTest {
val staleSocket = mockk<WebSocket>(relaxed = true)
every { staleSocket.send(any<String>()) } returns false
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, staleSocket)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.webSocketForTest = staleSocket
HermesWsClient.connectedForTest.set(true)
HermesWsClient.send(
WsMethods.PROMPT_SUBMIT,
mapOf("session_id" to "s1", "text" to "x".repeat(16 * 1024 * 1024 + 1)),
)
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
assertTrue(queue.isEmpty())
io.mockk.verify(exactly = 0) { staleSocket.cancel() }
}
@@ -674,10 +570,7 @@ class HermesWsClientTest {
mapOf("session_id" to "s1", "text" to "x".repeat(16 * 1024 * 1024 + 1)),
)
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
assertTrue(queue.isEmpty())
}
@@ -687,24 +580,14 @@ class HermesWsClientTest {
every { pressuredSocket.send(any<String>()) } returns false
every { pressuredSocket.queueSize() } returns 1024L
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, pressuredSocket)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.webSocketForTest = pressuredSocket
HermesWsClient.connectedForTest.set(true)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello"))
io.mockk.verify(exactly = 0) { pressuredSocket.cancel() }
val drainJobField = HermesWsClient::class.java.getDeclaredField("outboundDrainJob")
drainJobField.isAccessible = true
assertNotNull(drainJobField.get(HermesWsClient))
assertNotNull(HermesWsClient.outboundDrainJobForTest)
}
@Test
@@ -717,35 +600,18 @@ class HermesWsClientTest {
every { pressuredSocket.send(any<String>()) } returns false
every { pressuredSocket.queueSize() } returns 1024L
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, pressuredSocket)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1)
val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff")
backoffField.isAccessible = true
backoffField.setLong(HermesWsClient, 1_000L)
HermesWsClient.webSocketForTest = pressuredSocket
HermesWsClient.connectedForTest.set(true)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionGenerationForTest.set(1)
HermesWsClient.currentBackoffForTest = 1_000L
HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello"))
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val listener = constructor.newInstance(1) as WebSocketListener
val listener = HermesWsClient.createListenerForTest(1)
listener.onFailure(pressuredSocket, IOException("connection lost"), null)
assertEquals(2_000L, backoffField.getLong(HermesWsClient))
assertEquals(2_000L, HermesWsClient.currentBackoffForTest)
}
@Test
@@ -754,38 +620,22 @@ class HermesWsClientTest {
every { pressuredSocket.send(any<String>()) } returnsMany listOf(true, false)
every { pressuredSocket.queueSize() } returns 1024L
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
queue.add("{\"jsonrpc\":\"2.0\",\"id\":\"1\"}")
queue.add("{\"jsonrpc\":\"2.0\",\"id\":\"2\"}")
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionGenerationForTest.set(1)
HermesWsClient.webSocketForTest = pressuredSocket
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1)
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, pressuredSocket)
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val listener = constructor.newInstance(1) as WebSocketListener
val listener = HermesWsClient.createListenerForTest(1)
listener.onOpen(pressuredSocket, mockk(relaxed = true))
assertFalse(HermesWsClient.isConnected)
assertEquals(1, queue.size)
assertTrue(queue.peek()?.contains("\"2\"") == true)
io.mockk.verify(exactly = 0) { pressuredSocket.cancel() }
val drainJobField = HermesWsClient::class.java.getDeclaredField("outboundDrainJob")
drainJobField.isAccessible = true
assertNotNull(drainJobField.get(HermesWsClient))
assertNotNull(HermesWsClient.outboundDrainJobForTest)
}
@Test
@@ -794,29 +644,15 @@ class HermesWsClientTest {
val currentSocket = mockk<WebSocket>(relaxed = true)
every { currentSocket.send(any<String>()) } returns true
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, currentSocket)
HermesWsClient.webSocketForTest = currentSocket
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionGenerationForTest.set(2)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(2)
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
queue.add("{\"jsonrpc\":\"2.0\",\"id\":\"1\"}")
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val staleListener = constructor.newInstance(1) as WebSocketListener
val currentListener = constructor.newInstance(2) as WebSocketListener
val staleListener = HermesWsClient.createListenerForTest(1)
val currentListener = HermesWsClient.createListenerForTest(2)
val response = mockk<okhttp3.Response>(relaxed = true)
staleListener.onOpen(staleSocket, response)
@@ -835,40 +671,24 @@ class HermesWsClientTest {
false
}
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, staleSocket)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.webSocketForTest = staleSocket
HermesWsClient.connectedForTest.set(true)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello"))
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
assertTrue(queue.isEmpty())
assertEquals(ConnectionStatus.DISCONNECTED, HermesWsClient.connectionStatus.value)
}
@Test
fun testDisconnectPreservesQueuedMessagesUnlessExplicitlyCleared() {
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello"))
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
assertEquals(1, queue.size)
HermesWsClient.disconnect()
@@ -886,16 +706,11 @@ class HermesWsClientTest {
@Test
fun testRejectAllPendingRemovesQueuedAwaitedRpc() {
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.intentionalCloseForTest.set(false)
val deferred = HermesWsClient.request(WsMethods.PROCESS_LIST, mapOf("session_id" to "s1"))
val queueField = HermesWsClient::class.java.getDeclaredField("messageQueue")
queueField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val queue = queueField.get(HermesWsClient) as java.util.concurrent.ConcurrentLinkedQueue<String>
val queue = HermesWsClient.messageQueueForTest
assertEquals(1, queue.size)
HermesWsClient.rejectAllPending()
@@ -910,26 +725,12 @@ class HermesWsClientTest {
every { socket.send(any<String>()) } returns false
every { socket.queueSize() } returns 0L
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, socket)
HermesWsClient.webSocketForTest = socket
HermesWsClient.connectedForTest.set(true)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionGenerationForTest.set(1)
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1)
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val listener = constructor.newInstance(1) as WebSocketListener
val listener = HermesWsClient.createListenerForTest(1)
listener.onClosing(socket, 4401, "expired")
HermesWsClient.send(WsMethods.PROMPT_SUBMIT, mapOf("session_id" to "s1", "text" to "hello"))
@@ -941,21 +742,10 @@ class HermesWsClientTest {
@Test
fun testSocketFailureDoesNotOverwriteAuthExpired() {
val socket = mockk<WebSocket>(relaxed = true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1)
val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus")
statusField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val status = statusField.get(HermesWsClient) as MutableStateFlow<ConnectionStatus>
status.value = ConnectionStatus.AUTH_EXPIRED
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val listener = constructor.newInstance(1) as WebSocketListener
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionGenerationForTest.set(1)
HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED
val listener = HermesWsClient.createListenerForTest(1)
listener.onFailure(socket, IOException("network changed"), null)
@@ -965,18 +755,10 @@ class HermesWsClientTest {
@Test
fun testExplicitConnectRetriesAfterAuthExpired() =
runBlocking {
val statusField = HermesWsClient::class.java.getDeclaredField("_connectionStatus")
statusField.isAccessible = true
@Suppress("UNCHECKED_CAST")
val status = statusField.get(HermesWsClient) as MutableStateFlow<ConnectionStatus>
status.value = ConnectionStatus.AUTH_EXPIRED
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.connectionStatusForTest.value = ConnectionStatus.AUTH_EXPIRED
HermesWsClient.connectedForTest.set(true)
val oldSocket = mockk<WebSocket>(relaxed = true)
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, oldSocket)
HermesWsClient.webSocketForTest = oldSocket
mockWebServer.enqueue(MockResponse().withWebSocketUpgrade(object : WebSocketListener() {}))
HermesWsClient.connect()
@@ -1008,17 +790,16 @@ class HermesWsClientTest {
HermesWsClient.disconnect(clearPendingMessages = true)
mockWebServer.enqueue(MockResponse().withWebSocketUpgrade(object : WebSocketListener() {}))
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
val generation = generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger
val disconnectedGeneration = generation.get()
val disconnectedGeneration = HermesWsClient.connectionGenerationForTest.get()
val replacementConnect = thread { HermesWsClient.connect() }
val generationDeadline = System.currentTimeMillis() + 5_000
while (generation.get() == disconnectedGeneration && System.currentTimeMillis() < generationDeadline) {
while (HermesWsClient.connectionGenerationForTest.get() == disconnectedGeneration &&
System.currentTimeMillis() < generationDeadline
) {
Thread.sleep(10)
}
assertTrue(generation.get() > disconnectedGeneration)
val currentGeneration = generation.get()
assertTrue(HermesWsClient.connectionGenerationForTest.get() > disconnectedGeneration)
val currentGeneration = HermesWsClient.connectionGenerationForTest.get()
releaseRefresh.countDown()
staleConnect.join(5_000)
@@ -1029,7 +810,7 @@ class HermesWsClientTest {
assertFalse(staleConnect.isAlive)
assertFalse(replacementConnect.isAlive)
assertEquals(currentGeneration, generation.get())
assertEquals(currentGeneration, HermesWsClient.connectionGenerationForTest.get())
assertEquals(ConnectionStatus.CONNECTED, HermesWsClient.connectionStatus.value)
verify(exactly = 0) { AuthManager.setToken("stale-token") }
}
@@ -1441,12 +1222,10 @@ class HermesWsClientTest {
runBlocking { withTimeout(5000) { HermesWsClient.connectionStatus.first { it == ConnectionStatus.CONNECTED } } }
// After connect, backoff should be back to initial
val backoffField = HermesWsClient::class.java.getDeclaredField("currentBackoff")
backoffField.isAccessible = true
assertEquals(
"Backoff should reset to initial after successful connect",
1000L,
backoffField.getLong(HermesWsClient),
HermesWsClient.currentBackoffForTest,
)
}
@@ -1999,26 +1778,11 @@ class HermesWsClientTest {
fun testStreamingDedupPreservesSessionFallbackAndNumericSequenceSemantics() {
// Issue #1163: exercise the real listener, including the early duplicate return.
val socket = mockk<WebSocket>(relaxed = true)
val intentionalClose = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalClose.isAccessible = true
(intentionalClose.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
val generation = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generation.isAccessible = true
val constructor =
Class
.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
.declaredConstructors
.single()
constructor.isAccessible = true
val listener =
constructor.newInstance(
(generation.get(HermesWsClient) as AtomicInteger).get(),
) as WebSocketListener
HermesWsClient.intentionalCloseForTest.set(false)
val listener = HermesWsClient.createListenerForTest(HermesWsClient.connectionGenerationForTest.get())
// Directly install the reflection-built listener so onMessage runs even
// though this test never performed a real OkHttp connect.
val socketField = HermesWsClient::class.java.getDeclaredField("webSocket")
socketField.isAccessible = true
socketField.set(HermesWsClient, socket)
HermesWsClient.webSocketForTest = socket
mockkObject(EventParser)
val seqValues = listOf("6", "6.9", "4294967302", "\"6\"", "null", "true", "{}", "[]")
@@ -2039,18 +1803,10 @@ class HermesWsClientTest {
@Test
fun testTerminalCloseCode4403SetsAuthExpired() {
val socket = mockk<WebSocket>(relaxed = true)
val intentionalCloseField = HermesWsClient::class.java.getDeclaredField("intentionalClose")
intentionalCloseField.isAccessible = true
(intentionalCloseField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.intentionalCloseForTest.set(false)
HermesWsClient.connectionGenerationForTest.set(1)
val generationField = HermesWsClient::class.java.getDeclaredField("connectionGeneration")
generationField.isAccessible = true
(generationField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicInteger).set(1)
val listenerClass = Class.forName("com.m57.hermescontrol.data.ws.HermesWsClient\$WsListenerImpl")
val constructor = listenerClass.declaredConstructors.single()
constructor.isAccessible = true
val listener = constructor.newInstance(1) as WebSocketListener
val listener = HermesWsClient.createListenerForTest(1)
listener.onClosing(socket, 4403, "forbidden origin")
assertEquals(ConnectionStatus.AUTH_EXPIRED, HermesWsClient.connectionStatus.value)
@@ -2182,9 +1938,7 @@ class HermesWsClientTest {
@Test
fun testProbeLivenessOnTransportChangeCancelsSocketWhenPingFails() {
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(true)
HermesWsClient.connectedForTest.set(true)
val failureLatch = CountDownLatch(1)
HermesWsClient.probeLivenessOnTransportChange(
@@ -2199,9 +1953,7 @@ class HermesWsClientTest {
@Test
fun testProbeLivenessOnTransportChangeSkipsWhenDisconnected() {
val connectedField = HermesWsClient::class.java.getDeclaredField("connected")
connectedField.isAccessible = true
(connectedField.get(HermesWsClient) as java.util.concurrent.atomic.AtomicBoolean).set(false)
HermesWsClient.connectedForTest.set(false)
var failureInvoked = false
HermesWsClient.probeLivenessOnTransportChange(
@@ -43,7 +43,6 @@ import org.junit.Assert.assertTrue
import org.junit.Assert.fail
import org.junit.Before
import org.junit.Test
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicReference
@@ -122,13 +121,6 @@ class HermesWsClientTypedCallTest {
}
}
@Suppress("UNCHECKED_CAST")
private fun getPendingCalls(): ConcurrentHashMap<String, Any> {
val field = HermesWsClient::class.java.getDeclaredField("pendingCalls")
field.isAccessible = true
return field.get(HermesWsClient) as ConcurrentHashMap<String, Any>
}
@Test
fun callPutsExactFrameOnTheWire() {
val opened = CountDownLatch(1)
@@ -516,19 +508,25 @@ class HermesWsClientTypedCallTest {
assertTrue("Server should receive request", requestReceived.await(5, TimeUnit.SECONDS))
val pending = getPendingCalls()
assertEquals("Should have exactly 1 pending call while awaiting", 1, pending.size)
assertEquals(
"Should have exactly 1 pending call while awaiting",
1,
HermesWsClient.pendingCallIdsForTest().size,
)
callJob.cancel()
// Wait briefly for cancellation cleanup to execute
withTimeout(2000) {
while (pending.isNotEmpty()) {
while (HermesWsClient.pendingCallIdsForTest().isNotEmpty()) {
kotlinx.coroutines.delay(20)
}
}
assertTrue("Pending calls should be empty after caller coroutine is cancelled", pending.isEmpty())
assertTrue(
"Pending calls should be empty after caller coroutine is cancelled",
HermesWsClient.pendingCallIdsForTest().isEmpty(),
)
}
}
}
@@ -0,0 +1,98 @@
package com.m57.hermescontrol.data.ws
import org.junit.Assert.assertEquals
import org.junit.Test
class ReconnectPolicyTest {
@Test
fun defaultInitialBackoffIs1000() {
val policy = ReconnectPolicy()
assertEquals(1000L, policy.currentBackoff)
}
@Test
fun nextDelayMsUsesInjectedRandomAndFloorsAt300() {
// With random = 0.0, calculated delay is 0, floored to 300 (or ceiling if ceiling < 300)
val minPolicy = ReconnectPolicy(random = { 0.0 })
assertEquals(300L, minPolicy.nextDelayMs())
// With random = 0.5 and ceiling = 1000, 0.5 * 1000 = 500
val midPolicy = ReconnectPolicy(random = { 0.5 })
assertEquals(500L, midPolicy.nextDelayMs())
// With random nearly 1.0 (0.999), 0.999 * 1000 = 999
val maxPolicy = ReconnectPolicy(random = { 0.999 })
assertEquals(999L, maxPolicy.nextDelayMs())
}
@Test
fun nextDelayMsRespectsCeilingCapsBelowFloorAndZero() {
// Ceiling <= 0 results in 0
val zeroPolicy = ReconnectPolicy(random = { 0.5 })
zeroPolicy.currentBackoff = 0L
assertEquals(0L, zeroPolicy.nextDelayMs())
val negativePolicy = ReconnectPolicy(random = { 0.5 })
negativePolicy.currentBackoff = -100L
assertEquals(0L, negativePolicy.nextDelayMs())
// When ceiling is 100 (which is < 300 floor), delay should be capped to ceiling (100)
val lowCeilingPolicy = ReconnectPolicy(random = { 0.0 })
lowCeilingPolicy.currentBackoff = 100L
assertEquals(100L, lowCeilingPolicy.nextDelayMs())
}
@Test
fun advanceDoublesBackoffUpToCapOf30000() {
val policy = ReconnectPolicy()
assertEquals(1000L, policy.currentBackoff)
policy.advance()
assertEquals(2000L, policy.currentBackoff)
policy.advance()
assertEquals(4000L, policy.currentBackoff)
policy.advance()
assertEquals(8000L, policy.currentBackoff)
policy.advance()
assertEquals(16000L, policy.currentBackoff)
// Doubles past 30000 should cap exactly at 30000 (not speculative 15000 or 32000)
policy.advance()
assertEquals(30000L, policy.currentBackoff)
policy.advance()
assertEquals(30000L, policy.currentBackoff)
}
@Test
fun resetRestoresInitialBackoffOf1000() {
val policy = ReconnectPolicy()
policy.advance()
policy.advance()
assertEquals(4000L, policy.currentBackoff)
policy.reset()
assertEquals(1000L, policy.currentBackoff)
}
@Test
fun setBackoffForTestOverridesCurrentAndInitialBackoffDeterministically() {
val policy = ReconnectPolicy(random = { 0.5 })
policy.setBackoffForTest(1500L)
assertEquals(1500L, policy.currentBackoff)
// With test override, delay should be deterministic override value
assertEquals(1500L, policy.nextDelayMs())
// advance() still uses deterministic override if set, or advances from set initial/current
policy.advance()
assertEquals(1500L, policy.nextDelayMs())
// reset() retains deterministic override behavior or resets to overridden initial
policy.reset()
assertEquals(1500L, policy.nextDelayMs())
}
}
@@ -0,0 +1,555 @@
package com.m57.hermescontrol.data.ws
import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceParams
import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.Job
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.StandardTestDispatcher
import kotlinx.coroutines.test.TestScope
import kotlinx.coroutines.test.advanceUntilIdle
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
import java.util.concurrent.atomic.AtomicInteger
@OptIn(ExperimentalCoroutinesApi::class)
class ReplayTrackerTest {
private val json = Json { ignoreUnknownKeys = true }
private fun jsonElement(raw: String): JsonObject = json.parseToJsonElement(raw) as JsonObject
@Test
fun `initial state has empty watermarks and no replay in flight`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = {},
)
assertTrue(tracker.getSeqWatermarks().isEmpty())
assertFalse(tracker.isReplayInFlight())
}
@Test
fun `watermarks can be set read and cleared without cancelling jobs`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = {},
)
tracker.setSeqWatermark("s1", 10)
tracker.setSeqWatermark("s2", 25)
assertEquals(mapOf("s1" to 10, "s2" to 25), tracker.getSeqWatermarks())
tracker.clearSeqWatermarks()
assertTrue(tracker.getSeqWatermarks().isEmpty())
assertFalse(tracker.isReplayInFlight())
}
@Test
fun `triggerReplay is no-op when watermarks are empty`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
var fetchCalled = false
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = {
fetchCalled = true
SessionEventsSinceResult()
},
emitEvent = {},
)
tracker.triggerReplay()
assertFalse(tracker.isReplayInFlight())
testScope.advanceUntilIdle()
assertFalse(fetchCalled)
}
@Test
fun `triggerReplay arms hold synchronously and sets replay in flight before coroutine runs`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val deferred = CompletableDeferred<SessionEventsSinceResult>()
val emitted = mutableListOf<WsEvent>()
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { deferred.await() },
emitEvent = { emitted.add(it) },
)
tracker.setSeqWatermark("s1", 5)
// Arms hold synchronously
tracker.triggerReplay()
assertTrue(tracker.isReplayInFlight())
// Calling triggerReplay again while in flight is ignored
tracker.triggerReplay()
assertTrue(tracker.isReplayInFlight())
// A live event arriving now for s1 must be parked into hold without advancing watermark
val liveMsg = WsEvent.MessageToken(token = "live", sessionId = "s1")
tracker.acceptLiveEvent("s1", 8, liveMsg)
// Watermark has NOT advanced to 8 because replay is in flight and event is held
assertEquals(5, tracker.getSeqWatermarks()["s1"])
assertTrue(emitted.isEmpty())
// Complete replay
deferred.complete(SessionEventsSinceResult())
testScope.advanceUntilIdle()
// Replay completed, held event released and emitted, watermark advanced
assertFalse(tracker.isReplayInFlight())
assertEquals(listOf(liveMsg), emitted)
assertEquals(8, tracker.getSeqWatermarks()["s1"])
}
@Test
fun `isDuplicate drops equal or older seq and accepts newer seq`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = {},
)
tracker.setSeqWatermark("s1", 10)
// Same seq: duplicate
assertTrue(tracker.isDuplicate("s1", 10))
// Older seq: duplicate
assertTrue(tracker.isDuplicate("s1", 5))
// Newer seq: not duplicate
assertFalse(tracker.isDuplicate("s1", 11))
// Unknown session defaults to seq 0
assertFalse(tracker.isDuplicate("s2", 1))
assertTrue(tracker.isDuplicate("s2", 0))
}
@Test
fun `acceptLiveEvent without in-flight replay emits event and advances watermark`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val emitted = mutableListOf<WsEvent>()
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { emitted.add(it) },
)
tracker.setSeqWatermark("s1", 10)
val event = WsEvent.MessageToken(token = "tok", sessionId = "s1")
// Duplicate seq: dropped
tracker.acceptLiveEvent("s1", 10, event)
assertTrue(emitted.isEmpty())
assertEquals(10, tracker.getSeqWatermarks()["s1"])
// Newer seq: emitted and updates watermark
tracker.acceptLiveEvent("s1", 12, event)
assertEquals(listOf(event), emitted)
assertEquals(12, tracker.getSeqWatermarks()["s1"])
}
@Test
fun `replay events are emitted in provided order and advance watermarks`() =
runTest {
val emitted = mutableListOf<WsEvent>()
val event1 =
jsonElement(
"""{"type":"message.token","session_id":"s1","seq":6,"payload":{"text":"a"}}""",
)
val event2 =
jsonElement(
"""{"type":"message.token","session_id":"s1","seq":7,"payload":{"text":"b"}}""",
)
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = {
SessionEventsSinceResult(
events = listOf(event1, event2),
)
},
emitEvent = { emitted.add(it) },
)
tracker.setSeqWatermark("s1", 5)
tracker.fetchReplayForTest()
assertEquals(2, emitted.size)
assertEquals("a", (emitted[0] as WsEvent.MessageToken).token)
assertEquals("b", (emitted[1] as WsEvent.MessageToken).token)
assertEquals(7, tracker.getSeqWatermarks()["s1"])
assertFalse(tracker.isReplayInFlight())
}
@Test
fun `replay events with seq at or below watermark are dropped`() =
runTest {
val emitted = mutableListOf<WsEvent>()
val oldEvent =
jsonElement(
"""{"type":"message.token","session_id":"s1","seq":5,"payload":{"text":"old"}}""",
)
val newEvent =
jsonElement(
"""{"type":"message.token","session_id":"s1","seq":6,"payload":{"text":"new"}}""",
)
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = {
SessionEventsSinceResult(
events = listOf(oldEvent, newEvent),
)
},
emitEvent = { emitted.add(it) },
)
tracker.setSeqWatermark("s1", 5)
tracker.fetchReplayForTest()
assertEquals(1, emitted.size)
assertEquals("new", (emitted[0] as WsEvent.MessageToken).token)
assertEquals(6, tracker.getSeqWatermarks()["s1"])
}
@Test
fun `held live duplicates covered by replay are dropped and fresh held follow`() =
runTest {
val deferred = CompletableDeferred<SessionEventsSinceResult>()
val emitted = mutableListOf<WsEvent>()
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = { deferred.await() },
emitEvent = { emitted.add(it) },
)
tracker.setSeqWatermark("s1", 5)
tracker.triggerReplay()
// Live event 6 arrives while replay in flight
val live6 = WsEvent.MessageToken(token = "live6", sessionId = "s1")
val live7 = WsEvent.MessageToken(token = "live7", sessionId = "s1")
tracker.acceptLiveEvent("s1", 6, live6)
tracker.acceptLiveEvent("s1", 7, live7)
// Replay also returns event 6
val replay6 =
jsonElement(
"""{"type":"message.token","session_id":"s1","seq":6,"payload":{"text":"replay6"}}""",
)
deferred.complete(
SessionEventsSinceResult(
events = listOf(replay6),
),
)
testScheduler.advanceUntilIdle()
// Replay 6 is emitted first, advancing watermark to 6.
// Held live 6 (seq 6 <= 6) is dropped as duplicate.
// Held live 7 (seq 7 > 6) is emitted and advances watermark to 7.
assertEquals(2, emitted.size)
assertEquals("replay6", (emitted[0] as WsEvent.MessageToken).token)
assertEquals("live7", (emitted[1] as WsEvent.MessageToken).token)
assertEquals(7, tracker.getSeqWatermarks()["s1"])
assertFalse(tracker.isReplayInFlight())
}
@Test
fun `epoch change clears all watermarks updates epoch and requests transcript resync`() =
runTest {
val emitted = mutableListOf<WsEvent>()
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = {
SessionEventsSinceResult(
epoch = "epoch-2",
latestSeq = 50,
)
},
emitEvent = { emitted.add(it) },
)
tracker.setReplayEpochForTest("epoch-1")
tracker.setSeqWatermark("s1", 10)
tracker.setSeqWatermark("s2", 20)
tracker.fetchReplayForTest()
// Watermarks wiped
assertTrue(tracker.getSeqWatermarks().isEmpty())
assertEquals(1, emitted.size)
assertEquals(WsEvent.TranscriptResyncRequired("s1"), emitted[0])
}
@Test
fun `empty or null epoch in result does not trigger epoch change`() =
runTest {
val emitted = mutableListOf<WsEvent>()
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = {
SessionEventsSinceResult(
epoch = "",
events = emptyList(),
)
},
emitEvent = { emitted.add(it) },
)
tracker.setReplayEpochForTest("epoch-1")
tracker.setSeqWatermark("s1", 10)
tracker.fetchReplayForTest()
assertEquals(10, tracker.getSeqWatermarks()["s1"])
assertTrue(emitted.isEmpty())
}
@Test
fun `truncated resync fast-forwards watermark to latestSeq when epoch is same`() =
runTest {
val emitted = mutableListOf<WsEvent>()
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = {
SessionEventsSinceResult(
truncated = true,
latestSeq = 100,
epoch = "epoch-1",
)
},
emitEvent = { emitted.add(it) },
)
tracker.setReplayEpochForTest("epoch-1")
tracker.setSeqWatermark("s1", 10)
tracker.fetchReplayForTest()
// Fast-forwards to latestSeq and emits TranscriptResyncRequired
assertEquals(100, tracker.getSeqWatermarks()["s1"])
assertEquals(listOf(WsEvent.TranscriptResyncRequired("s1")), emitted)
}
@Test
fun `replay failure triggers resync invokes onReplayFailure and releases held events`() =
runTest {
val deferred = CompletableDeferred<SessionEventsSinceResult>()
val emitted = mutableListOf<WsEvent>()
val failedSessions = mutableListOf<Pair<String, Exception>>()
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = {
deferred.await()
},
emitEvent = { emitted.add(it) },
onReplayFailure = { sid, ex -> failedSessions.add(sid to ex) },
)
tracker.setSeqWatermark("s1", 10)
tracker.triggerReplay()
// Park a live event while waiting
val liveEvent = WsEvent.MessageToken(token = "held", sessionId = "s1")
tracker.acceptLiveEvent("s1", 15, liveEvent)
// Fail replay
val testException = IllegalStateException("network error")
deferred.completeExceptionally(testException)
testScheduler.advanceUntilIdle()
// Failure emitted TranscriptResyncRequired, reported to callback, and released held events
assertEquals(1, failedSessions.size)
assertEquals("s1", failedSessions[0].first)
assertTrue(failedSessions[0].second is IllegalStateException)
assertEquals("network error", failedSessions[0].second.message)
assertEquals(2, emitted.size)
assertEquals(WsEvent.TranscriptResyncRequired("s1"), emitted[0])
assertEquals(liveEvent, emitted[1])
assertEquals(15, tracker.getSeqWatermarks()["s1"])
assertFalse(tracker.isReplayInFlight())
}
@Test
fun `MessageComplete event uses resolveStoredSessionId injection`() =
runTest {
val emitted = mutableListOf<WsEvent>()
val msgCompleteJson =
jsonElement(
"""{"type":"message.complete","session_id":"wire-sid","payload":{"text":"done"}}""",
)
val tracker =
ReplayTracker(
scope = this,
fetchEventsSince = {
SessionEventsSinceResult(
events = listOf(msgCompleteJson),
)
},
emitEvent = { emitted.add(it) },
resolveStoredSessionId = { wireSid -> "resolved-$wireSid" },
)
tracker.setSeqWatermark("wire-sid", 5)
tracker.fetchReplayForTest()
assertEquals(1, emitted.size)
val complete = emitted[0] as WsEvent.MessageComplete
assertEquals("done", complete.text)
assertEquals("wire-sid", complete.sessionId)
assertEquals("resolved-wire-sid", complete.storedSessionId)
}
@Test
fun `updateEpoch clears watermarks when epoch changes`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = {},
)
tracker.setReplayEpochForTest("epoch-1")
tracker.setSeqWatermark("s1", 10)
// Same epoch does not clear
tracker.updateEpoch("epoch-1")
assertEquals(10, tracker.getSeqWatermarks()["s1"])
// Empty / null does not clear
tracker.updateEpoch(null)
tracker.updateEpoch("")
assertEquals(10, tracker.getSeqWatermarks()["s1"])
// Different epoch clears watermarks
tracker.updateEpoch("epoch-2")
assertTrue(tracker.getSeqWatermarks().isEmpty())
}
@Test
fun `clear cancels in-flight job and resets all state`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val deferred = CompletableDeferred<SessionEventsSinceResult>()
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { deferred.await() },
emitEvent = {},
)
tracker.setSeqWatermark("s1", 10)
tracker.triggerReplay()
assertTrue(tracker.isReplayInFlight())
tracker.clear()
assertFalse(tracker.isReplayInFlight())
assertTrue(tracker.getSeqWatermarks().isEmpty())
}
@Test
fun `acceptLiveEvent with eventFactory drops duplicate seq without invoking factory or emitting`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val emitted = mutableListOf<WsEvent>()
var factoryInvoked = false
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { emitted.add(it) },
)
tracker.setSeqWatermark("s1", 10)
// Duplicate seq <= watermark must not invoke factory nor emit
tracker.acceptLiveEvent(sessionId = "s1", seq = 10) {
factoryInvoked = true
WsEvent.MessageToken(token = "duplicate", sessionId = "s1")
}
assertFalse(factoryInvoked)
assertTrue(emitted.isEmpty())
assertEquals(10, tracker.getSeqWatermarks()["s1"])
}
@Test
fun `acceptLiveEvent with eventFactory invokes factory once and preserves single-check ordering`() {
val testDispatcher = StandardTestDispatcher()
val testScope = TestScope(testDispatcher)
val emitted = mutableListOf<WsEvent>()
val invocationCount = AtomicInteger(0)
val tracker =
ReplayTracker(
scope = testScope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { emitted.add(it) },
)
tracker.setSeqWatermark("s1", 10)
val expectedEvent = WsEvent.MessageToken(token = "fresh", sessionId = "s1")
// Factory intentionally injects an interleaved watermark advance (e.g. simulating race or side effect).
// Under single-check ordering, the initial pre-parse check (seq 11 > 10) succeeds, calls factory exactly once,
// and proceeds to emit and advance watermark without silently dropping due to an unexpected second check.
tracker.acceptLiveEvent(sessionId = "s1", seq = 11) {
invocationCount.incrementAndGet()
tracker.setSeqWatermark("s1", 99)
expectedEvent
}
assertEquals(1, invocationCount.get())
assertEquals(listOf(expectedEvent), emitted)
assertEquals(11, tracker.getSeqWatermarks()["s1"])
}
}
@@ -0,0 +1,790 @@
package com.m57.hermescontrol.data.ws
import com.m57.hermescontrol.data.ws.contract.RpcMethod
import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.advanceTimeBy
import kotlinx.coroutines.test.runCurrent
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.int
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertSame
import org.junit.Assert.assertTrue
import org.junit.Assert.fail
import org.junit.Test
@OptIn(ExperimentalCoroutinesApi::class)
class RpcChannelTest {
private val testJson =
Json {
ignoreUnknownKeys = true
encodeDefaults = false
}
@Serializable
private data class SampleRequestParams(
val message: String,
val count: Int? = null,
)
@Serializable
private data class SampleResult(
val status: String,
val total: Int,
)
private val sampleMethod =
RpcMethod("test.sample", SampleRequestParams.serializer(), SampleResult.serializer())
private class FakeGateway(
scope: CoroutineScope,
json: Json,
outboundLock: Any = Any(),
val onResultCallback: (String) -> Unit = {},
val onErrorCallback: (String) -> Unit = {},
resolveStoredSessionId: (String?) -> String? = { it },
val logParseFailureCallback: (Exception) -> Unit = {},
decorateParams: (String, Map<String, Any>) -> Map<String, Any> = { _, p -> p },
) {
val sentRequests = mutableListOf<SentRequestRecord>()
val sentFrames = mutableListOf<String>()
val removedQueuedIds = mutableListOf<String>()
val rejectedLogs = mutableListOf<Pair<String, String>>()
var idleCheckCount = 0
var sendFrameReturnValue = true
var immediateResolveOnSend: ((id: String, onSent: ((String) -> Unit)?) -> Unit)? = null
val channel: RpcChannel =
RpcChannel(
scope = scope,
json = json,
sendRequest = { method, params, onSent ->
val id = "req-${sentRequests.size + 1}"
sentRequests.add(SentRequestRecord(id, method, params))
onSent?.invoke(id)
immediateResolveOnSend?.invoke(id, onSent)
id
},
sendFrame = { frame ->
sentFrames.add(frame)
sendFrameReturnValue
},
removeQueuedMessage = { id ->
removedQueuedIds.add(id)
},
onIdleCheck = {
idleCheckCount++
},
logRejected = { method, id ->
rejectedLogs.add(method to id)
},
outboundLock = outboundLock,
onResult = onResultCallback,
onError = onErrorCallback,
resolveStoredSessionId = resolveStoredSessionId,
logParseFailure = logParseFailureCallback,
decorateParams = decorateParams,
)
data class SentRequestRecord(
val id: String,
val method: String,
val params: Map<String, Any>,
)
}
@Test
fun nextIdProducesIncreasingNonBlankStrings() {
val gateway = FakeGateway(CoroutineScope(CompletableDeferred<Unit>()), testJson)
val firstId = gateway.channel.nextId()
val secondId = gateway.channel.nextId()
assertEquals("1", firstId)
assertTrue(firstId.isNotBlank())
assertTrue(secondId.isNotBlank())
val firstNum = firstId.toLong()
val secondNum = secondId.toLong()
assertTrue(secondNum > firstNum)
assertEquals(firstNum + 1L, secondNum)
}
@Test
fun requestSuccessResolvesDeferredAndCleansUpInOrder() =
runTest {
val gateway = FakeGateway(this, testJson)
val deferred = gateway.channel.request("user.ping", mapOf("key" to "value"))
val id = gateway.sentRequests.first().id
assertTrue(gateway.channel.hasPendingCalls())
assertTrue(gateway.channel.pendingCallIdsForTest().contains(id))
assertNotNull(gateway.channel.pendingCallTimeoutForTest(id))
val dummyResult = JsonPrimitive("pong")
gateway.channel.resolvePending(id, dummyResult, null)
val resolved = deferred.await()
assertEquals(dummyResult, resolved)
assertFalse(gateway.channel.hasPendingCalls())
assertEquals(listOf(id), gateway.removedQueuedIds)
assertEquals(1, gateway.idleCheckCount)
assertNull(gateway.channel.pendingCallTimeoutForTest(id))
}
@Test
fun requestErrorResolvesWithHermesRpcExceptionAndErrorData() =
runTest {
val gateway = FakeGateway(this, testJson)
val deferred = gateway.channel.request("user.fail")
val id = gateway.sentRequests.first().id
val errorData = buildJsonObject { put("reason", "auth_failed") }
val rpcError = JsonRpcError(code = 401, message = "Unauthorized", data = errorData)
gateway.channel.resolvePending(id, null, rpcError)
try {
deferred.await()
fail("Expected HermesRpcException")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals("Unauthorized", e.message)
assertEquals(401, e.code)
assertEquals(errorData, e.data)
}
assertFalse(gateway.channel.hasPendingCalls())
assertEquals(listOf(id), gateway.removedQueuedIds)
assertEquals(1, gateway.idleCheckCount)
}
@Test
fun requestTimeoutFailsWithExactMessageAndCodeMinusOne() =
runTest {
val gateway = FakeGateway(this, testJson)
val timeoutMs = 5_000L
val deferred = gateway.channel.request("agent.turn", timeoutMs = timeoutMs)
val id = gateway.sentRequests.first().id
runCurrent()
advanceTimeBy(4_999L)
runCurrent()
assertTrue(gateway.channel.hasPendingCalls())
assertFalse(deferred.isCompleted)
advanceTimeBy(1L)
runCurrent()
assertTrue(deferred.isCompleted)
assertFalse(gateway.channel.hasPendingCalls())
assertEquals(listOf(id), gateway.removedQueuedIds)
assertEquals(1, gateway.idleCheckCount)
try {
deferred.await()
fail("Expected HermesRpcException on timeout")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals("Request timed out: agent.turn", e.message)
assertEquals(-1, e.code)
assertNull(e.data)
}
}
@Test
fun deferredCancellationCancelsTimerRemovesQueuedAndTriggersIdle() =
runTest {
val gateway = FakeGateway(this, testJson)
val deferred = gateway.channel.request("slow.method", timeoutMs = 60_000L)
val id = gateway.sentRequests.first().id
val timeoutJob = gateway.channel.pendingCallTimeoutForTest(id)
assertNotNull(timeoutJob)
assertFalse(timeoutJob!!.isCancelled)
deferred.cancel()
runCurrent()
assertTrue(deferred.isCancelled)
assertFalse(gateway.channel.hasPendingCalls())
assertTrue(timeoutJob.isCancelled)
assertEquals(listOf(id), gateway.removedQueuedIds)
assertEquals(1, gateway.idleCheckCount)
// Advancing past original timeout should do nothing further
advanceTimeBy(70_000L)
runCurrent()
assertEquals(1, gateway.idleCheckCount)
}
@Test
fun suppressionFlagReflectedInIsErrorEventSuppressed() =
runTest {
val gateway = FakeGateway(this, testJson)
gateway.channel.request("normal.call", suppressErrorEvent = false)
val normalId = gateway.sentRequests[0].id
gateway.channel.request("silent.call", suppressErrorEvent = true)
val silentId = gateway.sentRequests[1].id
assertFalse(gateway.channel.isErrorEventSuppressed(normalId))
assertTrue(gateway.channel.isErrorEventSuppressed(silentId))
assertFalse(gateway.channel.isErrorEventSuppressed("non-existent-id"))
}
@Test
fun rejectAllPendingFailsAllCallsWithProvidedExceptionAndNoIdleCheck() =
runTest {
val gateway = FakeGateway(this, testJson)
val deferred1 = gateway.channel.request("method.one")
val deferred2 = gateway.channel.request("method.two")
val id1 = gateway.sentRequests[0].id
val id2 = gateway.sentRequests[1].id
val customException = HermesWsClient.HermesRpcException("Socket terminated abruptly", code = 1006)
gateway.channel.rejectAllPending(customException)
assertFalse(gateway.channel.hasPendingCalls())
// Verified contract: rejectAllPending performs NO onIdleCheck
assertEquals(0, gateway.idleCheckCount)
assertEquals(setOf(id1, id2), gateway.removedQueuedIds.toSet())
assertEquals(setOf("method.one" to id1, "method.two" to id2), gateway.rejectedLogs.toSet())
try {
deferred1.await()
fail("deferred1 should have failed")
} catch (e: HermesWsClient.HermesRpcException) {
assertSame(customException, e)
}
try {
deferred2.await()
fail("deferred2 should have failed")
} catch (e: HermesWsClient.HermesRpcException) {
assertSame(customException, e)
}
}
@Test
fun rejectAllPendingDefaultExceptionMatchesExpectedMessage() =
runTest {
val gateway = FakeGateway(this, testJson)
val deferred = gateway.channel.request("some.call")
gateway.channel.rejectAllPending()
try {
deferred.await()
fail("Expected default exception")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals("Connection lost — request cancelled", e.message)
}
}
@Test
fun immediateResolutionDuringOnSentHandledSafely() =
runTest {
val gateway = FakeGateway(this, testJson)
gateway.immediateResolveOnSend = { id, _ ->
gateway.channel.resolvePending(id, "instant-result", null)
}
val deferred = gateway.channel.request("instant.call")
val id = gateway.sentRequests.first().id
assertTrue(deferred.isCompleted)
assertEquals("instant-result", deferred.await())
assertFalse(gateway.channel.hasPendingCalls())
assertEquals(listOf(id), gateway.removedQueuedIds)
assertEquals(1, gateway.idleCheckCount)
}
@Test
fun typedCallSerializesParamsOmittingDefaultsAndDecodesResult() =
runTest {
val gateway = FakeGateway(this, testJson)
val job =
async {
gateway.channel.call(
sampleMethod,
SampleRequestParams(message = "hello", count = null),
)
}
runCurrent()
assertEquals(1, gateway.sentRequests.size)
val sent = gateway.sentRequests.first()
assertEquals("test.sample", sent.method)
// encodeDefaults = false means count (null) is omitted from params
val encodedParams =
gateway.channel.encodeParams(
sampleMethod,
SampleRequestParams(message = "hello", count = null),
)
assertFalse(encodedParams.containsKey("count"))
assertTrue(encodedParams.containsKey("message"))
// Resolve pending with raw json representation
val rawResult =
buildJsonObject {
put("status", "ok")
put("total", 42)
put("extra_ignored_key", "bonus")
}
gateway.channel.resolvePending(sent.id, rawResult, null)
val result = job.await()
assertEquals("ok", result.status)
assertEquals(42, result.total)
}
@Test
fun typedCallCancellationPropagatesAndCancelsDeferred() =
runTest {
val gateway = FakeGateway(this, testJson)
val job =
launch {
gateway.channel.call(
sampleMethod,
SampleRequestParams(message = "cancel-me"),
)
}
runCurrent()
val id = gateway.sentRequests.first().id
assertTrue(gateway.channel.hasPendingCalls())
job.cancel()
runCurrent()
assertFalse(gateway.channel.hasPendingCalls())
assertEquals(listOf(id), gateway.removedQueuedIds)
assertEquals(1, gateway.idleCheckCount)
}
@Test
fun rawJsonNumericFidelityPreservedIntoResolve() =
runTest {
val gateway = FakeGateway(this, testJson)
val deferred =
gateway.channel.requestTyped(
sampleMethod,
SampleRequestParams(message = "precision"),
)
val id = gateway.sentRequests.first().id
// Map with integer-value double, e.g. 177.0 -> preserves int semantics in toJsonElement
val mapResult = mapOf("status" to "precise", "total" to 177.0)
gateway.channel.resolvePending(id, mapResult, null)
val raw = deferred.await()
val jsonElement = raw.toJsonElement() as JsonObject
assertEquals(177, jsonElement["total"]?.jsonPrimitive?.int)
// Direct decode with injected channel json to verify round-trip
val decoded = testJson.decodeFromJsonElement(sampleMethod.result, jsonElement)
assertEquals(177, decoded.total)
assertEquals("precise", decoded.status)
}
@Test
fun respondToServerRequestFormatsExactJsonRpcResponseAndPropagatesSink() {
val gateway = FakeGateway(CoroutineScope(CompletableDeferred<Unit>()), testJson)
val serverReqId = "srv-999"
val payload = buildJsonObject { put("pong", true) }
gateway.sendFrameReturnValue = true
val result = gateway.channel.respondToServerRequest(serverReqId, payload)
assertTrue(result)
assertEquals(1, gateway.sentFrames.size)
val frame = testJson.decodeFromString<JsonObject>(gateway.sentFrames.first())
assertEquals("2.0", frame["jsonrpc"]?.jsonPrimitive?.content)
assertEquals(serverReqId, frame["id"]?.jsonPrimitive?.content)
assertEquals(payload, frame["result"])
assertFalse(frame.containsKey("method"))
assertFalse(frame.containsKey("params"))
// Blank id returns false immediately without sending frame
val blankResult = gateway.channel.respondToServerRequest(" ", payload)
assertFalse(blankResult)
assertEquals(1, gateway.sentFrames.size)
// Sink returning false is propagated
gateway.sendFrameReturnValue = false
val falseResult = gateway.channel.respondToServerRequest("srv-1000", payload)
assertFalse(falseResult)
assertEquals(2, gateway.sentFrames.size)
}
@Test
fun respondToServerRequestErrorFormatsExactJsonRpcErrorAndPropagatesSink() {
val gateway = FakeGateway(CoroutineScope(CompletableDeferred<Unit>()), testJson)
val serverReqId = "srv-888"
gateway.sendFrameReturnValue = true
val result = gateway.channel.respondToServerRequestError(serverReqId, -32601, "Method not found")
assertTrue(result)
assertEquals(1, gateway.sentFrames.size)
val frame = testJson.decodeFromString<JsonObject>(gateway.sentFrames.first())
assertEquals("2.0", frame["jsonrpc"]?.jsonPrimitive?.content)
assertEquals(serverReqId, frame["id"]?.jsonPrimitive?.content)
assertFalse(frame.containsKey("result"))
val errorObj = frame["error"] as JsonObject
assertEquals(-32601, errorObj["code"]?.jsonPrimitive?.int)
assertEquals("Method not found", errorObj["message"]?.jsonPrimitive?.content)
// Blank id returns false without sending frame
assertFalse(gateway.channel.respondToServerRequestError("", -1, "Bad"))
assertEquals(1, gateway.sentFrames.size)
// Sink returning false is propagated
gateway.sendFrameReturnValue = false
val falseResult = gateway.channel.respondToServerRequestError("srv-889", -32000, "Error")
assertFalse(falseResult)
assertEquals(2, gateway.sentFrames.size)
}
@Test
fun createRequestDecoratesOnceAllocatesIdAndEncodesOmittingDefaults() {
var decorateCount = 0
val gateway =
FakeGateway(
scope = CoroutineScope(CompletableDeferred<Unit>()),
json = testJson,
decorateParams = { method, params ->
decorateCount++
params + ("decoratedMethod" to method)
},
)
val request = gateway.channel.createRequest("test.action", mapOf("key" to "value"))
assertEquals("1", request.id)
assertEquals("test.action", request.method)
assertEquals(1, decorateCount)
assertEquals(JsonPrimitive("value"), request.params["key"])
assertEquals(JsonPrimitive("test.action"), request.params["decoratedMethod"])
val encoded = gateway.channel.encodeRequest(request)
// Injected json with encodeDefaults = false omits default jsonrpc ("2.0")
assertFalse(encoded.contains("\"jsonrpc\""))
assertTrue(encoded.contains("\"id\":\"1\"") || encoded.contains("\"id\": \"1\""))
assertTrue(encoded.contains("\"method\":\"test.action\"") || encoded.contains("\"method\": \"test.action\""))
// An empty params map is omitted by encodeRequest under encodeDefaults = false
val plainGateway = FakeGateway(CoroutineScope(CompletableDeferred<Unit>()), testJson)
val emptyParamsReq = plainGateway.channel.createRequest("test.empty", emptyMap())
val encodedEmpty = plainGateway.channel.encodeRequest(emptyParamsReq)
assertFalse(encodedEmpty.contains("\"params\""))
}
@Test
fun handleIncomingFrameOrderRawJsonResolvedBeforeEventEmittedAndDoubleOnResult() =
runTest {
val onResultCalls = mutableListOf<String>()
val events = mutableListOf<WsEvent>()
val gateway =
FakeGateway(
scope = this,
json = testJson,
onResultCallback = { onResultCalls.add(it) },
)
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { events.add(it) },
)
val deferred = gateway.channel.request("client.calc")
val id = gateway.sentRequests.first().id
var rawResolvedAtEventTime: Boolean? = null
var onResultCallsAtEventTime = 0
val emitSink: (WsEvent) -> Unit = { event ->
events.add(event)
rawResolvedAtEventTime = deferred.isCompleted
onResultCallsAtEventTime = onResultCalls.size
}
val incomingFrame = """{"jsonrpc":"2.0","id":"$id","result":{"ans":42.0}}"""
gateway.channel.handleIncomingFrame(incomingFrame, replayTracker, emitSink)
// onResult is invoked twice baseline: once before raw resolve, and again before parsed RpcResult resolution
assertEquals(2, onResultCalls.size)
assertEquals(listOf(id, id), onResultCalls)
assertEquals(2, onResultCallsAtEventTime)
// Raw deferred was settled BEFORE emitEvent was called
assertEquals(true, rawResolvedAtEventTime)
assertTrue(deferred.isCompleted)
// Numeric fidelity preserved on result
val rawResult = deferred.await()
assertTrue(rawResult is JsonObject)
val rawObj = rawResult as JsonObject
assertEquals(JsonPrimitive(42.0), rawObj["ans"])
// Event emitted was RpcResult
assertEquals(1, events.size)
val rpcResult = events.first() as WsEvent.RpcResult
assertEquals(id, rpcResult.id)
}
@Test
fun handleIncomingFrameRpcErrorInvokesCallbackAndRespectsSuppression() =
runTest {
val onErrorCalls = mutableListOf<String>()
val events = mutableListOf<WsEvent>()
val gateway =
FakeGateway(
scope = this,
json = testJson,
onErrorCallback = { onErrorCalls.add(it) },
)
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { events.add(it) },
)
// 1. Normal unsuppressed error
val normalDeferred = gateway.channel.request("call.normal", suppressErrorEvent = false)
val normalId = gateway.sentRequests[0].id
val errorFrame1 =
"""{"jsonrpc":"2.0","id":"$normalId","error":{"code":-32603,"message":"Internal error"}}"""
gateway.channel.handleIncomingFrame(errorFrame1, replayTracker) { events.add(it) }
assertEquals(listOf(normalId), onErrorCalls)
assertTrue(normalDeferred.isCompleted)
try {
normalDeferred.await()
fail("Expected HermesRpcException")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals(-32603, e.code)
assertEquals("Internal error", e.message)
}
assertEquals(1, events.size)
assertTrue(events.first() is WsEvent.RpcError)
assertEquals(normalId, (events.first() as WsEvent.RpcError).id)
// 2. Suppressed error resolves deferred but emits NONE
val silentDeferred = gateway.channel.request("call.silent", suppressErrorEvent = true)
val silentId = gateway.sentRequests[1].id
val errorFrame2 =
"""{"jsonrpc":"2.0","id":"$silentId","error":{"code":404,"message":"Not found"}}"""
gateway.channel.handleIncomingFrame(errorFrame2, replayTracker) { events.add(it) }
assertEquals(listOf(normalId, silentId), onErrorCalls)
assertTrue(silentDeferred.isCompleted)
try {
silentDeferred.await()
fail("Expected HermesRpcException on silent deferred")
} catch (e: HermesWsClient.HermesRpcException) {
assertEquals(404, e.code)
}
// Event list size remains 1 (no RpcError event emitted for suppressed call)
assertEquals(1, events.size)
}
@Test
fun handleIncomingFrameCapabilitySuppressionHidesEventUntilCleared() =
runTest {
val events = mutableListOf<WsEvent>()
val gateway = FakeGateway(this, testJson)
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { events.add(it) },
)
val capId = "cap-42"
gateway.channel.registerCapabilityRequest(capId)
val capResponseFrame = """{"jsonrpc":"2.0","id":"$capId","result":{"supported":true}}"""
gateway.channel.handleIncomingFrame(capResponseFrame, replayTracker) { events.add(it) }
// Registered capability response is hidden from events
assertTrue(events.isEmpty())
// After clearing capabilities, subsequent responses with that ID are visible
gateway.channel.registerCapabilityRequest(capId)
gateway.channel.clearCapabilityRequests()
gateway.channel.handleIncomingFrame(capResponseFrame, replayTracker) { events.add(it) }
assertEquals(1, events.size)
val rpcResult = events.first() as WsEvent.RpcResult
assertEquals(capId, rpcResult.id)
}
@Test
fun handleIncomingFrameMalformedJsonInvokesParseFailureAndEmitsUnknown() =
runTest {
val parseFailures = mutableListOf<Exception>()
val events = mutableListOf<WsEvent>()
val gateway =
FakeGateway(
scope = this,
json = testJson,
logParseFailureCallback = { parseFailures.add(it) },
)
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { events.add(it) },
)
val malformed = "{ invalid json frame ::"
gateway.channel.handleIncomingFrame(malformed, replayTracker) { events.add(it) }
assertEquals(1, parseFailures.size)
assertEquals(1, events.size)
val unknown = events.first() as WsEvent.Unknown
assertEquals(malformed, unknown.raw)
}
@Test
fun handleIncomingFrameServerRequestClassifiedCorrectlyOverRpcResult() =
runTest {
val events = mutableListOf<WsEvent>()
val gateway = FakeGateway(this, testJson)
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { events.add(it) },
)
// Normal srq frame with id + method should emit ServerRequest, not RpcResult
val srqFrame = """{"jsonrpc":"2.0","id":"srq-101","method":"clarify","params":{"text":"Confirm?"}}"""
gateway.channel.handleIncomingFrame(srqFrame, replayTracker) { events.add(it) }
assertEquals(1, events.size)
val event = events.first()
assertTrue("Expected ServerRequest, got: $event", event is WsEvent.ServerRequest)
val srq = event as WsEvent.ServerRequest
assertEquals("srq-101", srq.id)
assertEquals("clarify", srq.method)
assertFalse(srq.replayed)
}
@Test
fun handleIncomingFrameSequencedNotificationDeduplicatesViaReplayTracker() =
runTest {
val events = mutableListOf<WsEvent>()
val gateway = FakeGateway(this, testJson)
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { events.add(it) },
)
val seqFrame =
"""{"method":"events","params":{"type":"message.token","session_id":"s1","seq":11,"payload":{"text":"fresh"}}}"""
// First delivery (fresh seq) emits once
gateway.channel.handleIncomingFrame(seqFrame, replayTracker) { events.add(it) }
assertEquals(1, events.size)
val tokenEvent = events.first() as WsEvent.MessageToken
assertEquals("fresh", tokenEvent.token)
assertEquals("s1", tokenEvent.sessionId)
// Duplicate delivery (same seq 11) is dropped, emits none
gateway.channel.handleIncomingFrame(seqFrame, replayTracker) { events.add(it) }
assertEquals(1, events.size)
}
@Test
fun handleIncomingFrameSettlesRawResultBeforeReplayingOpenRequests() =
runTest {
val order = mutableListOf<String>()
val gateway = FakeGateway(this, testJson, onResultCallback = { order.add("result:$it") })
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = {},
)
val deferred = gateway.channel.request("session.resume")
val id = gateway.sentRequests.first().id
deferred.invokeOnCompletion { order.add("resolved") }
val frame =
"""{"id":"$id","result":{"open_requests":[{"id":"srq-open","method":"clarify","params":{"text":"replayed"}}]}}"""
gateway.channel.handleIncomingFrame(frame, replayTracker) { event ->
assertTrue("Raw deferred must settle before either event", deferred.isCompleted)
when (event) {
is WsEvent.ServerRequest -> {
assertTrue(event.replayed)
assertEquals("clarify", event.method)
assertEquals("replayed", event.params["text"])
order.add("server:${event.id}")
}
is WsEvent.RpcResult -> {
order.add("rpc:${event.id}")
}
else -> {
fail("Unexpected event: $event")
}
}
}
assertEquals(listOf("result:$id", "resolved", "server:srq-open", "result:$id", "rpc:$id"), order)
assertTrue(deferred.await() is JsonObject)
assertFalse(gateway.channel.hasPendingCalls())
}
@Test
fun handleIncomingFrameDecoratesStoredSessionIdOnMessageComplete() =
runTest {
val events = mutableListOf<WsEvent>()
val gateway =
FakeGateway(
scope = this,
json = testJson,
resolveStoredSessionId = { sid -> "stored-$sid" },
)
val replayTracker =
ReplayTracker(
scope = this,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { events.add(it) },
)
val completeFrame =
"""{"method":"events","params":{"type":"message.complete","session_id":"active-session","payload":{"text":"done","reasoning":"reason"}}}"""
gateway.channel.handleIncomingFrame(completeFrame, replayTracker) { events.add(it) }
assertEquals(1, events.size)
val completeEvent = events.first() as WsEvent.MessageComplete
assertEquals("done", completeEvent.text)
assertEquals("reason", completeEvent.reasoning)
assertEquals("active-session", completeEvent.sessionId)
assertEquals("stored-active-session", completeEvent.storedSessionId)
}
}
@@ -0,0 +1,489 @@
package com.m57.hermescontrol.data.ws
import android.util.Log
import com.m57.hermescontrol.data.local.AuthManager
import com.m57.hermescontrol.data.remote.NetworkMonitor
import com.m57.hermescontrol.data.session.ActiveSessionHolder
import com.m57.hermescontrol.data.ws.contract.SessionEventsSinceResult
import io.mockk.every
import io.mockk.mockk
import io.mockk.mockkObject
import io.mockk.mockkStatic
import io.mockk.unmockkAll
import io.mockk.verify
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.test.TestScope
import kotlinx.coroutines.test.advanceTimeBy
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.json.Json
import okhttp3.Response
import okhttp3.WebSocket
import org.junit.After
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Assert.fail
import org.junit.Before
import org.junit.Test
import java.util.concurrent.atomic.AtomicInteger
@OptIn(ExperimentalCoroutinesApi::class)
class WsTransportTest {
private val testJson =
Json {
ignoreUnknownKeys = true
encodeDefaults = false
}
private class Fixture(
val scope: CoroutineScope,
json: Json,
nowMs: () -> Long = {
if (scope is TestScope) {
scope.testScheduler.currentTime
} else {
System.nanoTime() / 1_000_000L
}
},
) {
val outboundLock = Any()
val rawMessages = mutableListOf<String>()
val replayEmittedEvents = mutableListOf<WsEvent>()
val reconnectPolicy = ReconnectPolicy()
val rpcSentFrames = mutableListOf<String>()
val rpcRemovedQueueIds = mutableListOf<String>()
var rpcIdleChecks = 0
var transportRef: WsTransport? = null
private val rpcIdCounter = AtomicInteger(1)
val connectionHealth =
ConnectionHealth(
scope = scope,
isConnected = { transportRef?.isConnected ?: false },
pingRequest = { /* ping fake unit */ },
cancelSocket = { transportRef?.cancelSocket() },
pendingReply = { transportRef?.pendingReply ?: false },
nowMs = nowMs,
)
val replayTracker =
ReplayTracker(
scope = scope,
fetchEventsSince = { SessionEventsSinceResult() },
emitEvent = { replayEmittedEvents.add(it) },
)
val rpcChannel =
RpcChannel(
scope = scope,
json = json,
sendRequest = { method, params, onSent ->
val id = "rpc-${rpcIdCounter.getAndIncrement()}"
val req =
rpcChannelRef?.createRequest(method, params)?.copy(id = id)
?: JsonRpcRequest(id = id, method = method)
transportRef?.send(req, onSent) ?: id
},
sendFrame = { frame ->
rpcSentFrames.add(frame)
transportRef?.sendDirectFrame(frame) ?: false
},
removeQueuedMessage = { id ->
rpcRemovedQueueIds.add(id)
transportRef?.removeQueuedMessage(id)
},
onIdleCheck = {
rpcIdleChecks++
transportRef?.disconnectIfIdleInBackground()
},
outboundLock = outboundLock,
onResult = { id -> transportRef?.onRpcResult(id) },
onError = { id -> transportRef?.onRpcError(id) },
).also { rpcChannelRef = it }
val transport =
WsTransport(
scope = scope,
outboundLock = outboundLock,
reconnectPolicy = reconnectPolicy,
connectionHealth = connectionHealth,
replayTracker = replayTracker,
rpcChannel = rpcChannel,
onRawMessage = { rawMessages.add(it) },
).also { transportRef = it }
companion object {
private var rpcChannelRef: RpcChannel? = null
}
}
@Before
fun setUp() {
mockkStatic(Log::class)
every { Log.d(any<String>(), any<String>()) } returns 0
every { Log.i(any<String>(), any<String>()) } returns 0
every { Log.w(any<String>(), any<String>()) } returns 0
every { Log.w(any<String>(), any<String>(), any<Throwable>()) } returns 0
every { Log.e(any<String>(), any<String>(), any()) } returns 0
ActiveSessionHolder.clear()
mockkObject(AuthManager)
every { AuthManager.isAutoReconnect() } returns false
mockkObject(NetworkMonitor)
every { NetworkMonitor.isConnected } returns MutableStateFlow(true)
}
@After
fun tearDown() {
ActiveSessionHolder.clear()
unmockkAll()
}
@Test
fun sendAcceptCallbackRunsUnderOutboundLockAfterSocketAccept() =
runTest {
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send(any<String>()) } returns true
fixture.transport.connectedForTest.set(true)
fixture.transport.webSocketForTest = socket
var callbackRan = false
var callbackThreadHoldsLock = false
val request = JsonRpcRequest(id = "req-1", method = "session.list")
val id =
fixture.transport.send(
request = request,
onSent = {
callbackRan = true
callbackThreadHoldsLock = Thread.holdsLock(fixture.outboundLock)
},
)
assertEquals("req-1", id)
assertTrue(callbackRan)
assertTrue(callbackThreadHoldsLock)
verify(exactly = 1) { socket.send(any<String>()) }
assertTrue(fixture.transport.messageQueueForTest.isEmpty())
}
@Test
fun rejectedRetryableSendEnqueuesAndCancelDoesNotAutoRedialWhenAutoReconnectDisabled() =
runTest {
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send(any<String>()) } returns false
every { socket.queueSize() } returns 0L
fixture.transport.connectedForTest.set(true)
fixture.transport.intentionalCloseForTest.set(false)
fixture.transport.connectionGenerationForTest.set(1)
fixture.transport.webSocketForTest = socket
var onSentCalled = false
var callbackThreadHoldsLock = false
val request = JsonRpcRequest(id = "req-queued", method = "prompt.submit")
fixture.transport.send(
request = request,
onSent = {
onSentCalled = true
callbackThreadHoldsLock = Thread.holdsLock(fixture.outboundLock)
},
queueIfDisconnected = true,
)
// When socket rejects, request is pushed back into retryable queue
val queue = fixture.transport.messageQueueForTest
assertEquals(1, queue.size)
assertTrue(queue.peek()?.contains("req-queued") == true)
assertTrue(onSentCalled)
assertTrue(callbackThreadHoldsLock)
assertFalse(fixture.transport.isConnected)
assertEquals(ConnectionStatus.DISCONNECTED, fixture.transport.connectionStatus.value)
assertNull(fixture.transport.reconnectJobForTest)
verify(exactly = 1) { socket.cancel() }
}
@Test
fun staleOnMessageIgnoredWhileCurrentGenerationFrameUpdatesHealthAndRawCallback() =
runTest {
val fixture =
Fixture(
scope = this,
json = testJson,
nowMs = { testScheduler.currentTime },
)
val currentSocket = mockk<WebSocket>(relaxed = true)
val staleSocket = mockk<WebSocket>(relaxed = true)
fixture.transport.connectedForTest.set(true)
fixture.transport.connectionGenerationForTest.set(2)
fixture.transport.webSocketForTest = currentSocket
val initialTimestamp = fixture.connectionHealth.lastPongTimestamp
assertEquals(0L, initialTimestamp)
val staleListener = fixture.transport.createListenerForTest(1)
val currentListener = fixture.transport.createListenerForTest(2)
advanceTimeBy(1_000L)
// Message from stale listener/generation should be discarded
staleListener.onMessage(staleSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"old\"}")
staleListener.onMessage(currentSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"old-mismatched-gen\"}")
assertTrue(fixture.rawMessages.isEmpty())
assertEquals(0L, fixture.connectionHealth.lastPongTimestamp)
advanceTimeBy(1_000L)
// Message from current generation but mismatched socket should be discarded
currentListener.onMessage(staleSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"wrong-socket\"}")
assertTrue(fixture.rawMessages.isEmpty())
assertEquals(0L, fixture.connectionHealth.lastPongTimestamp)
advanceTimeBy(1_000L)
val expectedTimestamp = testScheduler.currentTime
// Matching generation and matching active socket passes through and updates health
currentListener.onMessage(currentSocket, "{\"jsonrpc\":\"2.0\",\"id\":\"valid\"}")
assertEquals(listOf("{\"jsonrpc\":\"2.0\",\"id\":\"valid\"}"), fixture.rawMessages)
assertEquals(expectedTimestamp, fixture.connectionHealth.lastPongTimestamp)
}
@Test
fun staleOnOpenClosesSupersededWithoutChangingOwnershipOrFlushingQueue() =
runTest {
val fixture = Fixture(this, testJson)
val staleSocket = mockk<WebSocket>(relaxed = true)
val currentSocket = mockk<WebSocket>(relaxed = true)
val response = mockk<Response>(relaxed = true)
fixture.transport.intentionalCloseForTest.set(false)
fixture.transport.connectionGenerationForTest.set(2)
fixture.transport.webSocketForTest = currentSocket
fixture.transport.messageQueueForTest.add("{\"jsonrpc\":\"2.0\",\"id\":\"queued-1\"}")
val staleListener = fixture.transport.createListenerForTest(1)
staleListener.onOpen(staleSocket, response)
// Stale onOpen must close the stale socket with 1000 ("Superseded")
verify(exactly = 1) { staleSocket.close(1000, "Superseded") }
// Must not change active socket or flush the queue on stale socket
verify(exactly = 0) { staleSocket.send(any<String>()) }
assertEquals(currentSocket, fixture.transport.webSocketForTest)
assertEquals(1, fixture.transport.messageQueueForTest.size)
}
@Test
fun authClosingCodeSetsTerminalAuthExpiredAndDoesNotAutoRedial() =
runTest {
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send(any<String>()) } returns true
fixture.transport.connectedForTest.set(true)
fixture.transport.intentionalCloseForTest.set(false)
fixture.transport.connectionGenerationForTest.set(1)
fixture.transport.webSocketForTest = socket
// Register a pending RPC call to verify it is NOT newly rejected
val pendingDeferred = fixture.rpcChannel.request("custom.in_flight", emptyMap())
assertFalse(pendingDeferred.isCompleted)
val listener = fixture.transport.createListenerForTest(1)
listener.onClosing(socket, 4401, "unauthorized")
assertEquals(ConnectionStatus.AUTH_EXPIRED, fixture.transport.connectionStatus.value)
assertTrue(fixture.transport.isConnected)
assertNull(fixture.transport.reconnectJobForTest)
assertFalse(pendingDeferred.isCompleted)
listener.onClosed(socket, 4401, "unauthorized")
assertFalse(fixture.transport.isConnected)
assertEquals(ConnectionStatus.AUTH_EXPIRED, fixture.transport.connectionStatus.value)
assertNull(fixture.transport.reconnectJobForTest)
assertFalse(pendingDeferred.isCompleted)
pendingDeferred.cancel()
}
@Test
fun disconnectPreservesPendingCallsWhileClearFlagControlsQueueAndLeaseCleanup() =
runTest {
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send(any<String>()) } returns true
fixture.transport.connectedForTest.set(true)
fixture.transport.intentionalCloseForTest.set(false)
fixture.transport.webSocketForTest = socket
// Register actual pending RPC call over socket accepting send
val pendingDeferred = fixture.rpcChannel.request("custom.in_flight", emptyMap())
assertFalse(pendingDeferred.isCompleted)
// Populate retryable message queue
fixture.transport.messageQueueForTest.add("{\"jsonrpc\":\"2.0\",\"id\":\"call-1\"}")
// Ordinary disconnect preserves queued messages and does NOT reject pending calls
fixture.transport.disconnect(clearPendingMessages = false)
assertEquals(1, fixture.transport.messageQueueForTest.size)
assertFalse(fixture.transport.isConnected)
assertFalse(pendingDeferred.isCompleted)
// Disconnect with clearPendingMessages = true clears queue and resets leases, still not rejecting pending calls
fixture.transport.acquireBackgroundConnectionLease()
assertTrue(fixture.transport.hasBackgroundConnectionLease())
fixture.transport.disconnect(clearPendingMessages = true)
assertTrue(fixture.transport.messageQueueForTest.isEmpty())
assertFalse(fixture.transport.hasBackgroundConnectionLease())
assertFalse(pendingDeferred.isCompleted)
pendingDeferred.cancel()
}
@Test
fun reconnectForNetworkChangeRejectsPendingCallsAndExecutesReplacementCallbackOnce() =
runTest {
every { AuthManager.isAutoReconnect() } returns true
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send(any<String>()) } returns true
fixture.transport.connectedForTest.set(true)
fixture.transport.intentionalCloseForTest.set(false)
fixture.transport.connectionGenerationForTest.set(1)
fixture.transport.webSocketForTest = socket
// Register non-empty pending RPC while socket accepts send
val pendingDeferred = fixture.rpcChannel.request("custom.network_call", emptyMap())
assertFalse(pendingDeferred.isCompleted)
var replacementOpens = 0
fixture.transport.reconnectForNetworkChange(
networkAvailable = true,
openReplacement = { replacementOpens++ },
)
// Outgoing socket canceled once
verify(exactly = 1) { socket.cancel() }
assertEquals(1, replacementOpens)
assertEquals(2, fixture.transport.connectionGenerationForTest.get())
assertFalse(fixture.transport.isConnected)
assertEquals(ConnectionStatus.RECONNECTING, fixture.transport.connectionStatus.value)
assertNull(fixture.transport.webSocketForTest)
// Network replacement uses the default connection-lost exception, not a timeout.
assertTrue(pendingDeferred.isCompleted)
try {
pendingDeferred.await()
fail("Expected HermesRpcException")
} catch (exception: HermesWsClient.HermesRpcException) {
assertEquals(0, exception.code)
assertEquals("Connection lost — request cancelled", exception.message)
}
}
@Test
fun combinedLeasesSurviveIdleBackgroundUntilBothReleased() =
runTest {
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
fixture.transport.connectedForTest.set(true)
fixture.transport.webSocketForTest = socket
fixture.transport.setConnectionStatusForTest(ConnectionStatus.CONNECTED)
fixture.transport.acquireExternalActivityConnectionLease()
fixture.transport.acquireBackgroundConnectionLease()
assertTrue(fixture.transport.hasBackgroundConnectionLease())
// Backgrounding app with both leases active keeps connection alive
fixture.transport.setAppForeground(false)
assertTrue(fixture.transport.isConnected)
// Releasing external lease keeps connection alive due to background lease
fixture.transport.releaseExternalActivityConnectionLease()
assertTrue(fixture.transport.isConnected)
// Releasing background lease releases all leases and disconnects idle background socket
fixture.transport.releaseBackgroundConnectionLease()
assertFalse(fixture.transport.isConnected)
assertEquals(ConnectionStatus.DISCONNECTED, fixture.transport.connectionStatus.value)
}
@Test
fun observeEventMarksPendingReplyStartAndCompleteRespectingIdlePolicy() =
runTest {
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
fixture.transport.connectedForTest.set(true)
fixture.transport.webSocketForTest = socket
fixture.transport.setConnectionStatusForTest(ConnectionStatus.CONNECTED)
// When a message start event arrives BEFORE setAppForeground(false)
val startEvent = WsEvent.MessageStart(sessionId = "s1")
fixture.transport.observeEvent(startEvent)
assertTrue(fixture.transport.pendingReply)
// Setting app foreground to false while pending reply is true stays connected
fixture.transport.setAppForeground(false)
assertTrue(fixture.transport.isConnected)
// Idle disconnect check while pending reply is true keeps socket connected
fixture.transport.disconnectIfIdleInBackground()
assertTrue(fixture.transport.isConnected)
// Message complete event clears pending reply and immediately idle disconnects
val completeEvent = WsEvent.MessageComplete(text = "done", sessionId = "s1")
fixture.transport.observeEvent(completeEvent)
assertFalse(fixture.transport.pendingReply)
assertFalse(fixture.transport.isConnected)
assertEquals(ConnectionStatus.DISCONNECTED, fixture.transport.connectionStatus.value)
}
@Test
fun sendDirectFrameBypassesQueueAndReturnsFalseWhenDisconnected() =
runTest {
val fixture = Fixture(this, testJson)
val socket = mockk<WebSocket>(relaxed = true)
every { socket.send("{\"jsonrpc\":\"2.0\",\"method\":\"connected_frame\"}") } returns true
every { socket.send("{\"jsonrpc\":\"2.0\",\"method\":\"send_fails\"}") } returns false
fixture.transport.connectedForTest.set(true)
fixture.transport.webSocketForTest = socket
// 1. Connected with socket accepting send -> returns true, no queue
val connectedResult =
fixture.transport.sendDirectFrame(
"{\"jsonrpc\":\"2.0\",\"method\":\"connected_frame\"}",
)
assertTrue(connectedResult)
assertTrue(fixture.transport.messageQueueForTest.isEmpty())
// 2. Connected with socket rejecting send -> returns false, no queue
val sendFailsResult = fixture.transport.sendDirectFrame("{\"jsonrpc\":\"2.0\",\"method\":\"send_fails\"}")
assertFalse(sendFailsResult)
assertTrue(fixture.transport.messageQueueForTest.isEmpty())
// 3. Disconnected -> returns false, no queue
fixture.transport.connectedForTest.set(false)
fixture.transport.webSocketForTest = null
val disconnectedResult = fixture.transport.sendDirectFrame("{\"jsonrpc\":\"2.0\",\"method\":\"direct\"}")
assertFalse(disconnectedResult)
assertTrue(fixture.transport.messageQueueForTest.isEmpty())
}
}