fix(bridge): stop replaying explicit terminal provider failures

This commit is contained in:
yilsem committed 2026-10-13 17:18:38 +03:00
1 parent 4037c65204
commit 4924781e2d
3 files changed
+313

No files matched your search

@@ -0,0 +1,119 @@
"""Transport-only handling of a worker's explicit terminal failure.
Never selects a model, evaluates content, or changes worker/Jev decisions.
If HTTP headers have already gone out, remember the failure for exactly one
identical retry. Codex then receives an HTTP error instead of re-running the
whole exhausted provider chain. Ordinary network interruptions remain retries.
"""
import hashlib
import json
import threading
import time
from collections import OrderedDict
def request_key(request, headers):
principal = "\0".join(headers.get(name, "") for name in
("Authorization", "ChatGPT-Account-Id", "session_id"))
body = json.dumps(request, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
return hashlib.sha256((principal + "\0" + body).encode()).digest()
class FailureReplays:
def __init__(self, ttl=120, capacity=128, clock=time.monotonic):
self.ttl, self.capacity, self.clock = ttl, capacity, clock
self.items = OrderedDict()
self.lock = threading.Lock()
def remember(self, key, error):
with self.lock:
self.items[key] = (self.clock() + self.ttl, error)
self.items.move_to_end(key)
while len(self.items) > self.capacity:
self.items.popitem(last=False)
def take(self, key):
with self.lock:
entry = self.items.pop(key, None)
return entry[1] if entry and entry[0] > self.clock() else None
def error_body(error):
# Preserve the actual provider failure; don't relabel it invalid_prompt,
# fabricate response.completed, or expose a rejected model response.
return {"error": {"type": "upstream_terminal_error", "code": error["code"],
"message": "Model yanıtı hata ile sonlandı. " + error["message"]}}
class Observer:
"""Bounded SSE observer. Forwarded stream bytes are never rewritten."""
LIMIT = 65536
def __init__(self):
self.tail = b""
self.data = []
self.size = 0
self.discard = False
self.discard_line = False
self.comment_seen = False
self.payload_seen = False
self.completed = False
self.error = None
def feed(self, chunk):
# read1() supplies small chunks, but also bound arbitrary caller input.
for offset in range(0, len(chunk), 4096):
self.tail += chunk[offset:offset + 4096]
while b"\n" in self.tail:
line, self.tail = self.tail.split(b"\n", 1)
if self.discard_line:
self.discard_line = False
continue
self._line(line.rstrip(b"\r"))
if len(self.tail) > self.LIMIT:
self.tail = b""
self.discard = self.discard_line = True
def _line(self, line):
if line.startswith(b":"):
self.comment_seen = True
elif line.startswith(b"data:"):
self.size += len(line)
if self.size > self.LIMIT:
self.discard = True
self.data.clear()
elif not self.discard:
self.data.append(line[5:].lstrip())
elif not line:
if self.data and not self.discard:
self._event(b"\n".join(self.data))
elif self.discard:
self.payload_seen = True
self.data, self.size, self.discard = [], 0, False
def _event(self, data):
try:
event = json.loads(data)
except (ValueError, UnicodeError):
self.payload_seen = True
return
if not isinstance(event, dict):
self.payload_seen = True
return
kind = event.get("type")
response = event.get("response")
error = event.get("error") if kind == "error" else response.get("error") if isinstance(response, dict) else None
# Known worker terminal errors only. Preserve context/rate-limit and
# other native error classifications rather than guessing semantics.
is_failure = (kind == "error" and isinstance(error, dict)
and error.get("type") == "upstream_error")
is_failure |= (kind == "response.failed" and isinstance(error, dict)
and error.get("code") == "jev_no_fallback")
if is_failure:
self.error = {"code": str(error.get("code") or "upstream_error")[:100],
"message": str(error.get("message") or "Sağlayıcı yanıtı tamamlanamadı.")[:2000]}
else:
self.payload_seen = True
if kind == "response.completed":
self.completed = True
self.error = None