fix(bridge): stop replaying explicit terminal provider failures
This commit is contained in:
yilsem committed
1 parent
4037c65204
commit
4924781e2d
3 files changed
+313
No files matched your search
@@ -0,0 +1,143 @@
|
||||
"""Real-socket regressions for terminal worker errors (no provider calls)."""
|
||||
import json
|
||||
import unittest
|
||||
|
||||
import test_ayirici as fixtures
|
||||
import terminal_stream
|
||||
|
||||
|
||||
ERROR = b'event: error\ndata: {"type":"error","error":{"type":"upstream_error","message":"upstream 503"}}\n\n'
|
||||
COMPLETE = b'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_ok","status":"completed","output":[]}}\n\n'
|
||||
|
||||
|
||||
class TerminalStream(unittest.TestCase):
|
||||
setUp = fixtures.Ayirici.setUp
|
||||
tearDown = fixtures.Ayirici.tearDown
|
||||
iste = fixtures.Ayirici.iste
|
||||
|
||||
def request(self, token="a", text="check"):
|
||||
body = json.dumps({"model": "ide/test", "input": [{"role": "user", "content": text}]})
|
||||
return self.iste("POST", "/backend-api/codex/responses", body,
|
||||
{"Authorization": "Bearer " + token})
|
||||
|
||||
def worker(self, payload):
|
||||
def respond(h, body):
|
||||
if h.path.endswith("/prepare-transfer"):
|
||||
return fixtures.duz(200, json.dumps({"request": json.loads(body)["request"]}).encode())(h, body)
|
||||
return fixtures.duz(200, payload, "text/event-stream")(h, body)
|
||||
self.kopru.cevap = respond
|
||||
|
||||
def inference_count(self):
|
||||
return sum(x["yol"].endswith("/responses") for x in self.kopru.gelen)
|
||||
|
||||
def test_error_only_stream_is_an_honest_http_failure(self):
|
||||
self.worker(ERROR)
|
||||
status, body, _ = self.request()
|
||||
self.assertEqual(status, 400)
|
||||
self.assertIn("upstream 503", json.loads(body)["error"]["message"])
|
||||
self.assertNotIn(b"response.completed", body)
|
||||
self.assertEqual(self.inference_count(), 1)
|
||||
|
||||
def test_error_after_heartbeat_stops_automatic_replay_once(self):
|
||||
def respond(h, body):
|
||||
if h.path.endswith("/prepare-transfer"):
|
||||
return fixtures.duz(200, json.dumps({"request": json.loads(body)["request"]}).encode())(h, body)
|
||||
h.send_response(200)
|
||||
h.send_header("Content-Type", "text/event-stream")
|
||||
h.send_header("Transfer-Encoding", "chunked")
|
||||
h.end_headers()
|
||||
for chunk in (b": keepalive\n\n", ERROR):
|
||||
h.wfile.write(b"%x\r\n" % len(chunk) + chunk + b"\r\n")
|
||||
h.wfile.flush()
|
||||
fixtures.time.sleep(.08)
|
||||
h.wfile.write(b"0\r\n\r\n")
|
||||
self.kopru.cevap = respond
|
||||
status, body, _ = self.request()
|
||||
self.assertEqual((status, body), (200, b": keepalive\n\n" + ERROR))
|
||||
status, body, _ = self.request()
|
||||
self.assertEqual(status, 400)
|
||||
self.assertIn(b"upstream 503", body)
|
||||
self.assertEqual(self.inference_count(), 1)
|
||||
# A later explicit retry is allowed; the one-shot guard is consumed.
|
||||
self.worker(COMPLETE)
|
||||
self.assertEqual(self.request()[:2], (200, COMPLETE))
|
||||
self.assertEqual(self.inference_count(), 2)
|
||||
|
||||
def test_native_error_stream_is_not_intercepted(self):
|
||||
self.ozgun.cevap = fixtures.duz(200, ERROR, "text/event-stream")
|
||||
body = json.dumps({"model": "gpt-6-astra", "input": []})
|
||||
self.assertEqual(self.iste("POST", "/backend-api/codex/responses", body)[:2], (200, ERROR))
|
||||
|
||||
def test_success_stream_is_byte_identical(self):
|
||||
self.worker(COMPLETE)
|
||||
self.assertEqual(self.request()[:2], (200, COMPLETE))
|
||||
self.assertEqual(self.request()[:2], (200, COMPLETE))
|
||||
self.assertEqual(self.inference_count(), 2)
|
||||
|
||||
def test_eof_without_upstream_error_keeps_network_retry(self):
|
||||
self.worker(b": keepalive\n\n")
|
||||
self.assertEqual(self.request()[0], 200)
|
||||
self.assertEqual(self.request()[0], 200)
|
||||
self.assertEqual(self.inference_count(), 2)
|
||||
|
||||
def test_worker_exhausted_failure_is_terminal_without_changing_its_reason(self):
|
||||
self.worker(b'event: response.failed\ndata: {"type":"response.failed","response":{"status":"failed","error":{"code":"jev_no_fallback","message":"No accepted response"}}}\n\n')
|
||||
status, body, _ = self.request()
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(json.loads(body)["error"]["code"], "jev_no_fallback")
|
||||
|
||||
def test_existing_context_error_classification_is_preserved(self):
|
||||
data = b'event: response.failed\ndata: {"type":"response.failed","response":{"error":{"code":"context_length_exceeded","message":"Context full"}}}\n\n'
|
||||
self.worker(data)
|
||||
self.assertEqual(self.request()[:2], (200, data))
|
||||
|
||||
|
||||
class ObserverTests(unittest.TestCase):
|
||||
def test_fragmented_crlf_and_multiline_json(self):
|
||||
data = b'event: error\r\ndata: {"type":"error",\r\ndata: "error":{"type":"upstream_error","message":"503"}}\r\n\r\n'
|
||||
observer = terminal_stream.Observer()
|
||||
for byte in data:
|
||||
observer.feed(bytes([byte]))
|
||||
self.assertEqual(observer.error['message'], '503')
|
||||
|
||||
def test_large_and_malformed_frames_are_bounded_and_do_not_crash(self):
|
||||
observer = terminal_stream.Observer()
|
||||
observer.feed(b'data: ' + b'x'*200000)
|
||||
self.assertLessEqual(len(observer.tail), observer.LIMIT)
|
||||
observer.feed(b'\n\ndata: {"response":[]}\n\n' + ERROR)
|
||||
self.assertEqual(observer.error['message'], 'upstream 503')
|
||||
|
||||
def test_completed_stream_does_not_poison_next_request(self):
|
||||
observer = terminal_stream.Observer()
|
||||
observer.feed(ERROR + COMPLETE)
|
||||
self.assertIsNone(observer.error)
|
||||
self.assertTrue(observer.completed)
|
||||
|
||||
def test_one_shot_expiry_and_capacity(self):
|
||||
now = [10]
|
||||
cache = terminal_stream.FailureReplays(ttl=5, capacity=2, clock=lambda: now[0])
|
||||
cache.remember(b'a', {'code': 'a'})
|
||||
self.assertEqual(cache.take(b'a'), {'code': 'a'})
|
||||
self.assertIsNone(cache.take(b'a'))
|
||||
cache.remember(b'a', {})
|
||||
now[0] = 16
|
||||
self.assertIsNone(cache.take(b'a'))
|
||||
for key in (b'a', b'b', b'c'):
|
||||
cache.remember(key, {'code': 'error'})
|
||||
self.assertIsNone(cache.take(b'a'))
|
||||
self.assertEqual(len(cache.items), 2)
|
||||
|
||||
def test_request_identity_separates_accounts_sessions_models_and_inputs(self):
|
||||
base = {'model': 'ide/a', 'input': []}
|
||||
headers = {'Authorization': 'Bearer a', 'session_id': 'one'}
|
||||
key = terminal_stream.request_key(base, headers)
|
||||
self.assertEqual(key, terminal_stream.request_key(dict(reversed(list(base.items()))), headers))
|
||||
for request, other in [({'model': 'ide/b', 'input': []}, headers),
|
||||
({'model': 'ide/a', 'input': ['next']}, headers),
|
||||
(base, dict(headers, Authorization='Bearer b')),
|
||||
(base, dict(headers, session_id='two'))]:
|
||||
self.assertNotEqual(key, terminal_stream.request_key(request, other))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user