211 lines
12 KiB
Python
211 lines
12 KiB
Python
"""Public Omni receipts. All scopes here are LOCAL opaque identities, never backend IDs."""
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import socket
|
|
import sqlite3
|
|
import ssl
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
from native_speech_mux import Binding, WatchReceiptAdapter
|
|
|
|
EVENTS_PATH = '/v1/bekci/events'
|
|
TRUSTED_ORIGIN = 'https://ai.turkservis.online'
|
|
# Public protocol lifetime, with bounded server/client clock skew. The server's
|
|
# expiry remains the watch deadline; there is no shorter research timeout.
|
|
MAX_RECEIPT_TTL = 24 * 60 * 60
|
|
RECEIPT_CLOCK_SKEW = 60
|
|
|
|
def opaque(*parts):
|
|
return hashlib.sha256('\0'.join(parts).encode()).hexdigest()
|
|
|
|
class Frames:
|
|
def __init__(self):
|
|
self.buf = b''
|
|
def feed(self, chunk):
|
|
self.buf += chunk
|
|
if len(self.buf) > 262144:
|
|
raise ValueError('SSE frame too large')
|
|
while True:
|
|
# Normalize only complete CRLF, including a CR split across reads.
|
|
self.buf = self.buf.replace(b'\r\n', b'\n')
|
|
if b'\n\n' not in self.buf:
|
|
return
|
|
frame, self.buf = self.buf.split(b'\n\n', 1)
|
|
event, data, cursor = '', [], None
|
|
for line in frame.decode('utf-8').split('\n'):
|
|
field, _, value = line.partition(':')
|
|
value = value.removeprefix(' ')
|
|
if field == 'event': event = value
|
|
elif field == 'data': data.append(value)
|
|
elif field == 'id': cursor = value
|
|
yield event, '\n'.join(data), cursor, frame + b'\n\n'
|
|
|
|
class MetadataFilter:
|
|
"""Strip sideband before rewrites, logging or transcript observers."""
|
|
def __init__(self, accept):
|
|
self.frames, self.accept = Frames(), accept
|
|
def feed(self, chunk):
|
|
out = []
|
|
for event, data, _, raw in self.frames.feed(chunk):
|
|
if event == 'bekci.watch':
|
|
try: self.accept(json.loads(data))
|
|
except (ValueError, TypeError): pass
|
|
else: out.append(raw)
|
|
return b''.join(out)
|
|
def flush(self):
|
|
# An incomplete sideband frame must never escape into model text.
|
|
raw, self.frames.buf = self.frames.buf, b''
|
|
return b'' if b'bekci.watch' in raw else raw
|
|
|
|
class Ledger:
|
|
"""Commit uncertain BEFORE enqueue. Crash ambiguity sacrifices delivery, never resends."""
|
|
def __init__(self, path):
|
|
self.db = sqlite3.connect(path, check_same_thread=False)
|
|
self.lock = threading.Lock()
|
|
self.db.execute('PRAGMA journal_mode=WAL')
|
|
self.db.execute('PRAGMA synchronous=FULL')
|
|
self.db.executescript('CREATE TABLE IF NOT EXISTS delivery(scope TEXT,id TEXT,status TEXT,PRIMARY KEY(scope,id)); CREATE TABLE IF NOT EXISTS cursors(scope TEXT PRIMARY KEY,cursor TEXT); CREATE TABLE IF NOT EXISTS bindings(scope TEXT PRIMARY KEY,epoch TEXT);')
|
|
def bind_epoch(self, scope, correlation):
|
|
epoch = opaque(correlation.connection_id,correlation.thread_id,str(correlation.generation))
|
|
with self.lock, self.db:
|
|
self.db.execute('INSERT OR IGNORE INTO bindings VALUES(?,?)',(scope,epoch))
|
|
return self.db.execute('SELECT epoch FROM bindings WHERE scope=?',(scope,)).fetchone()[0] == epoch
|
|
def claim(self, scope, nid):
|
|
with self.lock, self.db:
|
|
return self.db.execute('INSERT OR IGNORE INTO delivery VALUES(?,?,?)', (scope,nid,'uncertain')).rowcount == 1
|
|
def ack(self, scope, nid, cursor, status):
|
|
with self.lock, self.db:
|
|
self.db.execute('UPDATE delivery SET status=? WHERE scope=? AND id=?', (status,scope,nid))
|
|
self.db.execute('INSERT OR REPLACE INTO cursors VALUES(?,?)', (scope,cursor))
|
|
def cursor(self, scope):
|
|
with self.lock:
|
|
row = self.db.execute('SELECT cursor FROM cursors WHERE scope=?',(scope,)).fetchone()
|
|
return row[0] if row else None
|
|
|
|
class WatchOwner:
|
|
def __init__(self, mux, ledger, *, origin=TRUSTED_ORIGIN, test_loopback=False):
|
|
u = urlsplit(origin)
|
|
if origin != TRUSTED_ORIGIN and not (test_loopback and u.scheme == 'http' and u.hostname == '127.0.0.1'):
|
|
raise ValueError('untrusted Omni origin')
|
|
self.mux, self.ledger, self.origin = mux, ledger, u
|
|
self.adapter = WatchReceiptAdapter(mux)
|
|
self.lock, self.watches = threading.Lock(), {}
|
|
def capture(self, headers, body):
|
|
return self.adapter.capture(headers, body)
|
|
def live(self, c):
|
|
s = self.mux.snapshot(); t = s['threads'].get(c.thread_id, {})
|
|
return (not s['closed'] and not s['injection_disabled'] and s['connection_id'] == c.connection_id
|
|
and t.get('active') and t.get('generation') == c.generation)
|
|
def accept(self, payload, correlation, key, route_url):
|
|
if not correlation or not self.live(correlation): return 'stale_generation'
|
|
route = urlsplit(route_url)
|
|
if (route.scheme,route.netloc) != (self.origin.scheme,self.origin.netloc) or not key: return 'untrusted_route'
|
|
if (not isinstance(payload,dict) or set(payload) != {'subscription','events_url','expires_at'}
|
|
or payload['events_url'] != EVENTS_PATH or not isinstance(payload['subscription'],str)
|
|
or not 20 <= len(payload['subscription']) <= 128 or type(payload['expires_at']) not in (int,float)
|
|
or not time.time()*1000 < payload['expires_at'] <= (time.time()+MAX_RECEIPT_TTL+RECEIPT_CLOCK_SKEW)*1000): return 'invalid_receipt'
|
|
scope = opaque('public-watch',key,payload['subscription'])
|
|
if not self.ledger.bind_epoch(scope,correlation): return 'receipt_replay_mismatch'
|
|
with self.lock:
|
|
if scope in self.watches: return 'already_watching'
|
|
if len(self.watches) >= 256: return 'capacity_exhausted'
|
|
stop = threading.Event()
|
|
self.watches[scope] = stop
|
|
threading.Thread(target=self._watch,args=(dict(payload),correlation,key,scope,stop),daemon=True).start()
|
|
return 'subscribing'
|
|
def close(self):
|
|
with self.lock:
|
|
for stop in self.watches.values(): stop.set()
|
|
def _watch(self, p, c, key, scope, stop):
|
|
# These are local labels only. TLS+same-key authenticated ready attests
|
|
# receipt validity; server owns literal tenant/session/history mapping.
|
|
binding = Binding(opaque('member',key),scope,opaque('history-receipt',p['subscription']),c.connection_id,c.thread_id,c.generation)
|
|
deadline = p['expires_at']/1000
|
|
try:
|
|
backoff = 1
|
|
while not stop.is_set() and time.time() < deadline and self.live(c):
|
|
conn = (http.client.HTTPSConnection(self.origin.hostname,self.origin.port,context=ssl.create_default_context(),timeout=10)
|
|
if self.origin.scheme == 'https' else http.client.HTTPConnection(self.origin.hostname,self.origin.port,timeout=10))
|
|
finished = threading.Event()
|
|
transport, resp, guard_thread = None, None, None
|
|
def interrupt_transport():
|
|
# getresponse may detach conn.sock for HTTP/1.0 or
|
|
# Connection: close. Keep our actual socket until cleanup.
|
|
if transport is not None:
|
|
try: transport.shutdown(socket.SHUT_RDWR)
|
|
except OSError: pass
|
|
def guard():
|
|
while not finished.wait(.05):
|
|
if stop.is_set() or time.time() >= deadline or not self.live(c):
|
|
stop.set()
|
|
interrupt_transport()
|
|
return
|
|
try:
|
|
# Bound TCP/TLS connection establishment separately from
|
|
# headers/stream reads: quiet research has no idle deadline.
|
|
conn.connect()
|
|
transport = conn.sock
|
|
transport.settimeout(None)
|
|
if stop.is_set() or time.time() >= deadline or not self.live(c): break
|
|
guard_thread = threading.Thread(target=guard,daemon=True)
|
|
guard_thread.start()
|
|
body = {'subscription':p['subscription']}
|
|
cursor = self.ledger.cursor(scope)
|
|
if cursor: body['cursor'] = cursor
|
|
conn.request('POST', EVENTS_PATH, json.dumps(body), {'Authorization':'Bearer '+key,'Content-Type':'application/json'})
|
|
resp = conn.getresponse()
|
|
if resp.status != 200:
|
|
if resp.status not in (408,429) and not 500 <= resp.status < 600: break
|
|
elif 'text/event-stream' not in resp.getheader('Content-Type',''):
|
|
break
|
|
else:
|
|
parser, ready = Frames(), False
|
|
while self.live(c) and not stop.is_set() and time.time() < deadline:
|
|
chunk = resp.read1(4096)
|
|
if not chunk: break
|
|
for event,data,event_id,_ in parser.feed(chunk):
|
|
if stop.is_set() or time.time() >= deadline or not self.live(c): break
|
|
if not data: continue # keepalive
|
|
value = json.loads(data)
|
|
if event == 'ready':
|
|
if not isinstance(value,dict) or not isinstance(value.get('cursor'),str): raise ValueError('invalid ready')
|
|
ready = self.live(c) and self.mux.bind(binding)
|
|
if not ready: return
|
|
elif event == 'notice':
|
|
if not ready: raise ValueError('notice before ready')
|
|
self._notice(binding,scope,value,event_id,deadline,c)
|
|
except (OSError,ValueError,http.client.HTTPException):
|
|
pass # no credential-bearing exceptions in logs
|
|
finally:
|
|
finished.set()
|
|
interrupt_transport()
|
|
if guard_thread is not None: guard_thread.join()
|
|
if resp is not None: resp.close()
|
|
conn.close()
|
|
if transport is not None: transport.close()
|
|
if stop.is_set() or time.time() >= deadline or not self.live(c): break
|
|
if stop.wait(min(backoff,max(0,deadline-time.time()))): break
|
|
backoff = min(backoff*2,15)
|
|
finally:
|
|
with self.mux._lock:
|
|
k = (binding.tenant_id,binding.bekci_session_id)
|
|
if self.mux._bindings.get(k) == binding: self.mux._bindings.pop(k,None)
|
|
with self.lock: self.watches.pop(scope,None)
|
|
def _notice(self,b,scope,value,event_id,deadline,c):
|
|
if not self.live(c) or not isinstance(value,dict): return
|
|
n, cursor = value.get('notice'), value.get('cursor')
|
|
if (not isinstance(n,dict) or not isinstance(cursor,str) or not 0 < len(cursor) <= 512
|
|
or event_id != cursor or not isinstance(n.get('id'),str) or not 0 < len(n['id']) <= 256
|
|
or n.get('type') not in ('duzeltme','arastirma') or not isinstance(n.get('metin'),str)
|
|
or not 0 < len(n['metin']) <= 8000): return
|
|
if not self.ledger.claim(scope,n['id']):
|
|
self.ledger.ack(scope,n['id'],cursor,'replay_suppressed'); return
|
|
now = time.time()
|
|
notice = dict(notice_id=n['id'],tenant_id=b.tenant_id,bekci_session_id=b.bekci_session_id,
|
|
history_revision=b.history_revision,connection_id=b.connection_id,thread_id=b.thread_id,
|
|
expected_generation=b.generation,issued_at=now,expires_at=min(now+60,deadline),text=n['metin'])
|
|
result = self.mux.submit(notice)
|
|
self.ledger.ack(scope,n['id'],cursor,result['status'])
|