fix(bridge): stop replaying explicit terminal provider failures
This commit is contained in:
1 parent
4037c65204
commit
4924781e2d
3 files changed
+313
No files matched your search
@@ -0,0 +1,119 @@
|
||||
"""Transport-only handling of a worker's explicit terminal failure.
|
||||
|
||||
Never selects a model, evaluates content, or changes worker/Jev decisions.
|
||||
If HTTP headers have already gone out, remember the failure for exactly one
|
||||
identical retry. Codex then receives an HTTP error instead of re-running the
|
||||
whole exhausted provider chain. Ordinary network interruptions remain retries.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def request_key(request, headers):
|
||||
principal = "\0".join(headers.get(name, "") for name in
|
||||
("Authorization", "ChatGPT-Account-Id", "session_id"))
|
||||
body = json.dumps(request, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
return hashlib.sha256((principal + "\0" + body).encode()).digest()
|
||||
|
||||
|
||||
class FailureReplays:
|
||||
def __init__(self, ttl=120, capacity=128, clock=time.monotonic):
|
||||
self.ttl, self.capacity, self.clock = ttl, capacity, clock
|
||||
self.items = OrderedDict()
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def remember(self, key, error):
|
||||
with self.lock:
|
||||
self.items[key] = (self.clock() + self.ttl, error)
|
||||
self.items.move_to_end(key)
|
||||
while len(self.items) > self.capacity:
|
||||
self.items.popitem(last=False)
|
||||
|
||||
def take(self, key):
|
||||
with self.lock:
|
||||
entry = self.items.pop(key, None)
|
||||
return entry[1] if entry and entry[0] > self.clock() else None
|
||||
|
||||
|
||||
def error_body(error):
|
||||
# Preserve the actual provider failure; don't relabel it invalid_prompt,
|
||||
# fabricate response.completed, or expose a rejected model response.
|
||||
return {"error": {"type": "upstream_terminal_error", "code": error["code"],
|
||||
"message": "Model yanıtı hata ile sonlandı. " + error["message"]}}
|
||||
|
||||
|
||||
class Observer:
|
||||
"""Bounded SSE observer. Forwarded stream bytes are never rewritten."""
|
||||
LIMIT = 65536
|
||||
|
||||
def __init__(self):
|
||||
self.tail = b""
|
||||
self.data = []
|
||||
self.size = 0
|
||||
self.discard = False
|
||||
self.discard_line = False
|
||||
self.comment_seen = False
|
||||
self.payload_seen = False
|
||||
self.completed = False
|
||||
self.error = None
|
||||
|
||||
def feed(self, chunk):
|
||||
# read1() supplies small chunks, but also bound arbitrary caller input.
|
||||
for offset in range(0, len(chunk), 4096):
|
||||
self.tail += chunk[offset:offset + 4096]
|
||||
while b"\n" in self.tail:
|
||||
line, self.tail = self.tail.split(b"\n", 1)
|
||||
if self.discard_line:
|
||||
self.discard_line = False
|
||||
continue
|
||||
self._line(line.rstrip(b"\r"))
|
||||
if len(self.tail) > self.LIMIT:
|
||||
self.tail = b""
|
||||
self.discard = self.discard_line = True
|
||||
|
||||
def _line(self, line):
|
||||
if line.startswith(b":"):
|
||||
self.comment_seen = True
|
||||
elif line.startswith(b"data:"):
|
||||
self.size += len(line)
|
||||
if self.size > self.LIMIT:
|
||||
self.discard = True
|
||||
self.data.clear()
|
||||
elif not self.discard:
|
||||
self.data.append(line[5:].lstrip())
|
||||
elif not line:
|
||||
if self.data and not self.discard:
|
||||
self._event(b"\n".join(self.data))
|
||||
elif self.discard:
|
||||
self.payload_seen = True
|
||||
self.data, self.size, self.discard = [], 0, False
|
||||
|
||||
def _event(self, data):
|
||||
try:
|
||||
event = json.loads(data)
|
||||
except (ValueError, UnicodeError):
|
||||
self.payload_seen = True
|
||||
return
|
||||
if not isinstance(event, dict):
|
||||
self.payload_seen = True
|
||||
return
|
||||
kind = event.get("type")
|
||||
response = event.get("response")
|
||||
error = event.get("error") if kind == "error" else response.get("error") if isinstance(response, dict) else None
|
||||
# Known worker terminal errors only. Preserve context/rate-limit and
|
||||
# other native error classifications rather than guessing semantics.
|
||||
is_failure = (kind == "error" and isinstance(error, dict)
|
||||
and error.get("type") == "upstream_error")
|
||||
is_failure |= (kind == "response.failed" and isinstance(error, dict)
|
||||
and error.get("code") == "jev_no_fallback")
|
||||
if is_failure:
|
||||
self.error = {"code": str(error.get("code") or "upstream_error")[:100],
|
||||
"message": str(error.get("message") or "Sağlayıcı yanıtı tamamlanamadı.")[:2000]}
|
||||
else:
|
||||
self.payload_seen = True
|
||||
if kind == "response.completed":
|
||||
self.completed = True
|
||||
self.error = None
|
||||
Reference in new issue
Block a user