144 lines
6.7 KiB
Python
144 lines
6.7 KiB
Python
"""Real-socket regressions for terminal worker errors (no provider calls)."""
|
|
import json
|
|
import unittest
|
|
|
|
import test_ayirici as fixtures
|
|
import terminal_stream
|
|
|
|
|
|
ERROR = b'event: error\ndata: {"type":"error","error":{"type":"upstream_error","message":"upstream 503"}}\n\n'
|
|
COMPLETE = b'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_ok","status":"completed","output":[]}}\n\n'
|
|
|
|
|
|
class TerminalStream(unittest.TestCase):
|
|
setUp = fixtures.Ayirici.setUp
|
|
tearDown = fixtures.Ayirici.tearDown
|
|
iste = fixtures.Ayirici.iste
|
|
|
|
def request(self, token="a", text="check"):
|
|
body = json.dumps({"model": "ide/test", "input": [{"role": "user", "content": text}]})
|
|
return self.iste("POST", "/backend-api/codex/responses", body,
|
|
{"Authorization": "Bearer " + token})
|
|
|
|
def worker(self, payload):
|
|
def respond(h, body):
|
|
if h.path.endswith("/prepare-transfer"):
|
|
return fixtures.duz(200, json.dumps({"request": json.loads(body)["request"]}).encode())(h, body)
|
|
return fixtures.duz(200, payload, "text/event-stream")(h, body)
|
|
self.kopru.cevap = respond
|
|
|
|
def inference_count(self):
|
|
return sum(x["yol"].endswith("/responses") for x in self.kopru.gelen)
|
|
|
|
def test_error_only_stream_is_an_honest_http_failure(self):
|
|
self.worker(ERROR)
|
|
status, body, _ = self.request()
|
|
self.assertEqual(status, 400)
|
|
self.assertIn("upstream 503", json.loads(body)["error"]["message"])
|
|
self.assertNotIn(b"response.completed", body)
|
|
self.assertEqual(self.inference_count(), 1)
|
|
|
|
def test_error_after_heartbeat_stops_automatic_replay_once(self):
|
|
def respond(h, body):
|
|
if h.path.endswith("/prepare-transfer"):
|
|
return fixtures.duz(200, json.dumps({"request": json.loads(body)["request"]}).encode())(h, body)
|
|
h.send_response(200)
|
|
h.send_header("Content-Type", "text/event-stream")
|
|
h.send_header("Transfer-Encoding", "chunked")
|
|
h.end_headers()
|
|
for chunk in (b": keepalive\n\n", ERROR):
|
|
h.wfile.write(b"%x\r\n" % len(chunk) + chunk + b"\r\n")
|
|
h.wfile.flush()
|
|
fixtures.time.sleep(.08)
|
|
h.wfile.write(b"0\r\n\r\n")
|
|
self.kopru.cevap = respond
|
|
status, body, _ = self.request()
|
|
self.assertEqual((status, body), (200, b": keepalive\n\n" + ERROR))
|
|
status, body, _ = self.request()
|
|
self.assertEqual(status, 400)
|
|
self.assertIn(b"upstream 503", body)
|
|
self.assertEqual(self.inference_count(), 1)
|
|
# A later explicit retry is allowed; the one-shot guard is consumed.
|
|
self.worker(COMPLETE)
|
|
self.assertEqual(self.request()[:2], (200, COMPLETE))
|
|
self.assertEqual(self.inference_count(), 2)
|
|
|
|
def test_native_error_stream_is_not_intercepted(self):
|
|
self.ozgun.cevap = fixtures.duz(200, ERROR, "text/event-stream")
|
|
body = json.dumps({"model": "gpt-6-astra", "input": []})
|
|
self.assertEqual(self.iste("POST", "/backend-api/codex/responses", body)[:2], (200, ERROR))
|
|
|
|
def test_success_stream_is_byte_identical(self):
|
|
self.worker(COMPLETE)
|
|
self.assertEqual(self.request()[:2], (200, COMPLETE))
|
|
self.assertEqual(self.request()[:2], (200, COMPLETE))
|
|
self.assertEqual(self.inference_count(), 2)
|
|
|
|
def test_eof_without_upstream_error_keeps_network_retry(self):
|
|
self.worker(b": keepalive\n\n")
|
|
self.assertEqual(self.request()[0], 200)
|
|
self.assertEqual(self.request()[0], 200)
|
|
self.assertEqual(self.inference_count(), 2)
|
|
|
|
def test_worker_exhausted_failure_is_terminal_without_changing_its_reason(self):
|
|
self.worker(b'event: response.failed\ndata: {"type":"response.failed","response":{"status":"failed","error":{"code":"jev_no_fallback","message":"No accepted response"}}}\n\n')
|
|
status, body, _ = self.request()
|
|
self.assertEqual(status, 400)
|
|
self.assertEqual(json.loads(body)["error"]["code"], "jev_no_fallback")
|
|
|
|
def test_existing_context_error_classification_is_preserved(self):
|
|
data = b'event: response.failed\ndata: {"type":"response.failed","response":{"error":{"code":"context_length_exceeded","message":"Context full"}}}\n\n'
|
|
self.worker(data)
|
|
self.assertEqual(self.request()[:2], (200, data))
|
|
|
|
|
|
class ObserverTests(unittest.TestCase):
|
|
def test_fragmented_crlf_and_multiline_json(self):
|
|
data = b'event: error\r\ndata: {"type":"error",\r\ndata: "error":{"type":"upstream_error","message":"503"}}\r\n\r\n'
|
|
observer = terminal_stream.Observer()
|
|
for byte in data:
|
|
observer.feed(bytes([byte]))
|
|
self.assertEqual(observer.error['message'], '503')
|
|
|
|
def test_large_and_malformed_frames_are_bounded_and_do_not_crash(self):
|
|
observer = terminal_stream.Observer()
|
|
observer.feed(b'data: ' + b'x'*200000)
|
|
self.assertLessEqual(len(observer.tail), observer.LIMIT)
|
|
observer.feed(b'\n\ndata: {"response":[]}\n\n' + ERROR)
|
|
self.assertEqual(observer.error['message'], 'upstream 503')
|
|
|
|
def test_completed_stream_does_not_poison_next_request(self):
|
|
observer = terminal_stream.Observer()
|
|
observer.feed(ERROR + COMPLETE)
|
|
self.assertIsNone(observer.error)
|
|
self.assertTrue(observer.completed)
|
|
|
|
def test_one_shot_expiry_and_capacity(self):
|
|
now = [10]
|
|
cache = terminal_stream.FailureReplays(ttl=5, capacity=2, clock=lambda: now[0])
|
|
cache.remember(b'a', {'code': 'a'})
|
|
self.assertEqual(cache.take(b'a'), {'code': 'a'})
|
|
self.assertIsNone(cache.take(b'a'))
|
|
cache.remember(b'a', {})
|
|
now[0] = 16
|
|
self.assertIsNone(cache.take(b'a'))
|
|
for key in (b'a', b'b', b'c'):
|
|
cache.remember(key, {'code': 'error'})
|
|
self.assertIsNone(cache.take(b'a'))
|
|
self.assertEqual(len(cache.items), 2)
|
|
|
|
def test_request_identity_separates_accounts_sessions_models_and_inputs(self):
|
|
base = {'model': 'ide/a', 'input': []}
|
|
headers = {'Authorization': 'Bearer a', 'session_id': 'one'}
|
|
key = terminal_stream.request_key(base, headers)
|
|
self.assertEqual(key, terminal_stream.request_key(dict(reversed(list(base.items()))), headers))
|
|
for request, other in [({'model': 'ide/b', 'input': []}, headers),
|
|
({'model': 'ide/a', 'input': ['next']}, headers),
|
|
(base, dict(headers, Authorization='Bearer b')),
|
|
(base, dict(headers, session_id='two'))]:
|
|
self.assertNotEqual(key, terminal_stream.request_key(request, other))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|