fix(voice): keep watcher alive through quiet checks and reconnects
This commit is contained in:
1 parent
b4e8664115
commit
2718b460ed
2 files changed
+146
-25
No files matched your search
@@ -124,48 +124,70 @@ class WatchOwner:
|
||||
binding = Binding(opaque('member',key),scope,opaque('history-receipt',p['subscription']),c.connection_id,c.thread_id,c.generation)
|
||||
deadline = p['expires_at']/1000
|
||||
try:
|
||||
for attempt in range(8):
|
||||
if stop.is_set() or time.time() >= deadline or not self.live(c): break
|
||||
backoff = 1
|
||||
while not stop.is_set() and time.time() < deadline and self.live(c):
|
||||
conn = (http.client.HTTPSConnection(self.origin.hostname,self.origin.port,context=ssl.create_default_context(),timeout=10)
|
||||
if self.origin.scheme == 'https' else http.client.HTTPConnection(self.origin.hostname,self.origin.port,timeout=10))
|
||||
finished = threading.Event()
|
||||
transport, resp, guard_thread = None, None, None
|
||||
def interrupt_transport():
|
||||
# getresponse may detach conn.sock for HTTP/1.0 or
|
||||
# Connection: close. Keep our actual socket until cleanup.
|
||||
if transport is not None:
|
||||
try: transport.shutdown(socket.SHUT_RDWR)
|
||||
except OSError: pass
|
||||
def guard():
|
||||
while not finished.wait(.05):
|
||||
if stop.is_set() or time.time() >= deadline or not self.live(c):
|
||||
stop.set()
|
||||
if conn.sock:
|
||||
try: conn.sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError: pass
|
||||
conn.close()
|
||||
interrupt_transport()
|
||||
return
|
||||
guard_thread = threading.Thread(target=guard,daemon=True)
|
||||
guard_thread.start()
|
||||
try:
|
||||
# Bound TCP/TLS connection establishment separately from
|
||||
# headers/stream reads: quiet research has no idle deadline.
|
||||
conn.connect()
|
||||
transport = conn.sock
|
||||
transport.settimeout(None)
|
||||
if stop.is_set() or time.time() >= deadline or not self.live(c): break
|
||||
guard_thread = threading.Thread(target=guard,daemon=True)
|
||||
guard_thread.start()
|
||||
body = {'subscription':p['subscription']}
|
||||
cursor = self.ledger.cursor(scope)
|
||||
if cursor: body['cursor'] = cursor
|
||||
conn.request('POST', EVENTS_PATH, json.dumps(body), {'Authorization':'Bearer '+key,'Content-Type':'application/json'})
|
||||
resp = conn.getresponse()
|
||||
if resp.status != 200 or 'text/event-stream' not in resp.getheader('Content-Type',''): break
|
||||
parser, ready = Frames(), False
|
||||
while self.live(c) and not stop.is_set() and time.time() < deadline:
|
||||
chunk = resp.read1(4096)
|
||||
if not chunk: break
|
||||
for event,data,event_id,_ in parser.feed(chunk):
|
||||
if not data: continue # keepalive
|
||||
value = json.loads(data)
|
||||
if event == 'ready':
|
||||
if not isinstance(value,dict) or not isinstance(value.get('cursor'),str): raise ValueError('invalid ready')
|
||||
ready = self.live(c) and self.mux.bind(binding)
|
||||
if not ready: return
|
||||
elif event == 'notice':
|
||||
if not ready: raise ValueError('notice before ready')
|
||||
self._notice(binding,scope,value,event_id,deadline,c)
|
||||
if resp.status != 200:
|
||||
if resp.status not in (408,429) and not 500 <= resp.status < 600: break
|
||||
elif 'text/event-stream' not in resp.getheader('Content-Type',''):
|
||||
break
|
||||
else:
|
||||
parser, ready = Frames(), False
|
||||
while self.live(c) and not stop.is_set() and time.time() < deadline:
|
||||
chunk = resp.read1(4096)
|
||||
if not chunk: break
|
||||
for event,data,event_id,_ in parser.feed(chunk):
|
||||
if stop.is_set() or time.time() >= deadline or not self.live(c): break
|
||||
if not data: continue # keepalive
|
||||
value = json.loads(data)
|
||||
if event == 'ready':
|
||||
if not isinstance(value,dict) or not isinstance(value.get('cursor'),str): raise ValueError('invalid ready')
|
||||
ready = self.live(c) and self.mux.bind(binding)
|
||||
if not ready: return
|
||||
elif event == 'notice':
|
||||
if not ready: raise ValueError('notice before ready')
|
||||
self._notice(binding,scope,value,event_id,deadline,c)
|
||||
except (OSError,ValueError,http.client.HTTPException):
|
||||
pass # no credential-bearing exceptions in logs
|
||||
finally:
|
||||
finished.set(); conn.close(); guard_thread.join(timeout=1)
|
||||
if stop.wait(min(2**attempt,15)): break
|
||||
finished.set()
|
||||
interrupt_transport()
|
||||
if guard_thread is not None: guard_thread.join()
|
||||
if resp is not None: resp.close()
|
||||
conn.close()
|
||||
if transport is not None: transport.close()
|
||||
if stop.is_set() or time.time() >= deadline or not self.live(c): break
|
||||
if stop.wait(min(backoff,max(0,deadline-time.time()))): break
|
||||
backoff = min(backoff*2,15)
|
||||
finally:
|
||||
with self.mux._lock:
|
||||
k = (binding.tenant_id,binding.bekci_session_id)
|
||||
|
||||
Reference in new issue
Block a user