feat(settings): configure Jev evaluation and model transitions
This commit is contained in:
1 parent
369f6c2ca8
commit
883b1dc824
20 files changed
+1195
-37
No files matched your search
@@ -34,7 +34,7 @@ import urllib.request
|
||||
import gecis_context
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SURUM = "11" # AITURK IDE'nin bekledigi kopru surumu (codex-link-config.ts CODEX_BRIDGE_VERSION)
|
||||
SURUM = "12" # AITURK IDE'nin bekledigi kopru surumu (codex-link-config.ts CODEX_BRIDGE_VERSION)
|
||||
try:
|
||||
with open(os.path.abspath(__file__), "rb") as _f:
|
||||
KOD = hashlib.sha256(_f.read()).hexdigest()[:8]
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"enabled": true,
|
||||
"endpoint": "https://api.typesafe.ai/v1/systemone",
|
||||
"model": "jev-1.13.0",
|
||||
"fallbackModel": "Katman-1",
|
||||
"rejectionRules": [
|
||||
{ "from": "Katman-1", "to": "Katman-2" },
|
||||
{ "from": "Katman-2", "to": "Katman-3" },
|
||||
{ "from": "Katman-3", "to": "Katman-4" }
|
||||
],
|
||||
"exemptModels": ["Sesli-Hizli-Zeki", "Katman-4"],
|
||||
"thresholds": { "ret": 0.5, "sart": 0.7 },
|
||||
"questions": {
|
||||
"ret": "The assistant did NOT do what the user asked or did NOT answer the user's question: it refused, declined, said it cannot or will not, avoided giving the requested judgment, or redirected to a different topic.",
|
||||
"sart": "The assistant changed the request on its own: it added its own conditions, softened or relabeled what was asked, or left part of it out, instead of doing exactly what the user asked."
|
||||
},
|
||||
"timeoutSeconds": 3600,
|
||||
"criteria": { "ret": { "true": "", "false": "" }, "sart": { "true": "", "false": "" } },
|
||||
"maxConcurrent": 0,
|
||||
"queueWaitSeconds": 3600,
|
||||
"failureAction": "pass"
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
import jev_settings
|
||||
|
||||
ADRES = "https://api.typesafe.ai/v1/systemone"
|
||||
MODEL = "jev-1.13.0"
|
||||
@@ -32,26 +33,44 @@ SORU = {
|
||||
_ANAHTAR = {"okundu": False, "deger": ""}
|
||||
|
||||
def muaf_mi(hedef):
|
||||
"""12 Ekim 2026: yalnız çözülmüş exact sesli combo hedefi Jev'den muaftır.
|
||||
"""Settings' always-pass list matches exact resolved model IDs.
|
||||
|
||||
Taşıma hatalarının katman sırasını değiştirmez; yedek model kendi adına denetlenir.
|
||||
"""
|
||||
return hedef == "Sesli-Hizli-Zeki"
|
||||
return hedef in jev_settings.read()[0]['exemptModels']
|
||||
|
||||
|
||||
def sonrakiler(hedef):
|
||||
"""Jev gecirmezse sirayla denenecek katmanlar. Secili model bir katmansa ondan sonrakiler; katman degilse
|
||||
butun katmanlar bastan (once Katman-1). Son katman (Katman-4) kendisi gelince bos doner: sorgulanmadan gecer."""
|
||||
"""Model-specific rejection edges take precedence over the general fallback.
|
||||
|
||||
Stop at an always-pass model or before revisiting any target.
|
||||
"""
|
||||
settings = jev_settings.read()[0]
|
||||
ad = str(hedef or "")
|
||||
if not ad or ad == KATMAN[-1]:
|
||||
if not settings['enabled'] or not ad:
|
||||
return []
|
||||
if ad in KATMAN:
|
||||
return list(KATMAN[KATMAN.index(ad) + 1:])
|
||||
return list(KATMAN)
|
||||
routes = {r['from']: r['to'] for r in settings['rejectionRules']}
|
||||
result, seen = [], {ad}
|
||||
# An exempt model can still use the general fallback on transport failure.
|
||||
# Repeated targets stop the walk even if a file was edited outside Settings.
|
||||
while len(result) < 65:
|
||||
target = routes.get(ad, settings['fallbackModel'])
|
||||
if target in seen:
|
||||
break
|
||||
seen.add(target)
|
||||
result.append(target)
|
||||
if target in settings['exemptModels']:
|
||||
break
|
||||
ad = target
|
||||
return result
|
||||
|
||||
|
||||
def anahtar(ortam=None, dosyalar=None):
|
||||
"""TypeSafe anahtari: ortam degiskeni, sonra ~/.typesafe/typesafe.env. Yoksa ""."""
|
||||
if ortam is None and dosyalar is None:
|
||||
saved = jev_settings.key()
|
||||
if saved:
|
||||
return saved
|
||||
ortam = os.environ if ortam is None else ortam
|
||||
deger = str(ortam.get("TYPESAFE_API_KEY") or "").strip()
|
||||
if deger:
|
||||
@@ -147,24 +166,53 @@ def komutlar(govde):
|
||||
return "\n".join(satirlar)
|
||||
|
||||
|
||||
def _gonder(govde, key):
|
||||
istek = urllib.request.Request(ADRES, data=json.dumps(govde).encode("utf-8"), method="POST",
|
||||
def _gonder(govde, key, settings=None):
|
||||
settings = settings or jev_settings.read()[0]
|
||||
istek = urllib.request.Request(settings['endpoint'], data=json.dumps(govde).encode("utf-8"), method="POST",
|
||||
headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(istek) as yanit:
|
||||
with urllib.request.urlopen(istek, timeout=settings['timeoutSeconds']) as yanit:
|
||||
return json.load(yanit)
|
||||
|
||||
|
||||
class Puan(dict):
|
||||
"""Carry the thresholds used for this evaluation through a settings change."""
|
||||
pass
|
||||
|
||||
|
||||
def sor(istek, cevap, key, gonder=_gonder):
|
||||
"""Jev'e sorar. Donus: {"ret", "sart"} olasiliklari | None (sorulamadi)."""
|
||||
govde = {"model": MODEL, "state": {"user_request": istek, "assistant_reply": cevap},
|
||||
"questions": {k: {"type": "noul", "instructions": v} for k, v in SORU.items()}}
|
||||
settings = jev_settings.read()[0]
|
||||
govde = {"model": settings['model'], "state": {"user_request": istek, "assistant_reply": cevap},
|
||||
"questions": {k: {"type": "noul", "instructions": v} for k, v in settings['questions'].items()}}
|
||||
for name, criteria in settings.get('criteria', {}).items():
|
||||
values = {k: v for k, v in criteria.items() if v.strip()}
|
||||
if values:
|
||||
govde['questions'][name]['criteria'] = values
|
||||
if not jev_settings.acquire(settings):
|
||||
return None
|
||||
try:
|
||||
yanitlar = (gonder(govde, key) or {}).get("answers") or {}
|
||||
return {k: float(yanitlar[k]["noul"]) for k in SORU}
|
||||
response = gonder(govde, key, settings) if gonder is _gonder else gonder(govde, key)
|
||||
yanitlar = (response or {}).get("answers") or {}
|
||||
result = Puan({k: float(yanitlar[k]["noul"]) for k in SORU})
|
||||
if any(not 0 <= v <= 1 for v in result.values()):
|
||||
return None
|
||||
result.thresholds = settings['thresholds']
|
||||
return result
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
jev_settings.release()
|
||||
|
||||
|
||||
def karar(puan):
|
||||
"""True: adim gecmez, sonraki katman yapar. Istisna yok."""
|
||||
return puan["ret"] >= ESIK["ret"] or puan["sart"] >= ESIK["sart"]
|
||||
thresholds = getattr(puan, 'thresholds', None) or jev_settings.read()[0]['thresholds']
|
||||
return puan["ret"] >= thresholds["ret"] or puan["sart"] >= thresholds["sart"]
|
||||
|
||||
|
||||
def hata_gecis():
|
||||
return jev_settings.read()[0]['failureAction'] == 'next'
|
||||
|
||||
|
||||
def etkin_mi():
|
||||
return bool(jev_settings.read()[0]['enabled'])
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Per-user settings, re-read for new evaluations without restarting active chats."""
|
||||
import base64
|
||||
import copy
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULTS = json.loads(Path(__file__).with_name('jev-settings-defaults.json').read_text(encoding='utf-8'))
|
||||
_condition = threading.Condition()
|
||||
_active = 0
|
||||
|
||||
|
||||
def home():
|
||||
if '--home' in sys.argv:
|
||||
return Path(sys.argv[sys.argv.index('--home') + 1])
|
||||
return Path(os.environ.get('AITURK_CODEX_BRIDGE_HOME') or Path(__file__).parent)
|
||||
|
||||
|
||||
def read():
|
||||
try:
|
||||
data = json.loads((home() / 'jev-settings.json').read_text(encoding='utf-8'))
|
||||
except FileNotFoundError:
|
||||
return copy.deepcopy(DEFAULTS), None
|
||||
settings = data['settings']
|
||||
# A corrupted file must not silently change routing or contact another host.
|
||||
from urllib.parse import urlsplit
|
||||
url = urlsplit(settings['endpoint'])
|
||||
if url.scheme != 'https' or not url.netloc or url.username or url.password:
|
||||
raise ValueError('Invalid Jev endpoint')
|
||||
for name in ('ret', 'sart'):
|
||||
n = settings['thresholds'][name]
|
||||
if not isinstance(n, (int, float)) or not 0 <= n <= 1 or not settings['questions'][name]:
|
||||
raise ValueError('Invalid Jev rule')
|
||||
for name in ('timeoutSeconds', 'queueWaitSeconds'):
|
||||
if not isinstance(settings[name], int) or not 1 <= settings[name] <= 3600:
|
||||
raise ValueError('Invalid Jev timeout')
|
||||
if not isinstance(settings['maxConcurrent'], int) or not 0 <= settings['maxConcurrent'] <= 512:
|
||||
raise ValueError('Invalid Jev concurrency')
|
||||
if not settings['fallbackModel'] or not isinstance(settings['rejectionRules'], list) or settings['failureAction'] not in ('pass', 'next'):
|
||||
raise ValueError('Invalid Jev routing')
|
||||
return settings, data.get('protectedKey')
|
||||
|
||||
|
||||
def key():
|
||||
_, encrypted = read()
|
||||
if not encrypted:
|
||||
return ''
|
||||
if os.name != 'nt':
|
||||
raise RuntimeError('Windows credential protection required')
|
||||
from ctypes import wintypes
|
||||
class Blob(ctypes.Structure):
|
||||
_fields_ = [('cbData', wintypes.DWORD), ('pbData', ctypes.POINTER(ctypes.c_ubyte))]
|
||||
raw = base64.b64decode(encrypted, validate=True)
|
||||
buf = (ctypes.c_ubyte * len(raw)).from_buffer_copy(raw)
|
||||
source, result = Blob(len(raw), buf), Blob()
|
||||
decrypt = ctypes.windll.crypt32.CryptUnprotectData
|
||||
decrypt.argtypes = [ctypes.POINTER(Blob), ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, wintypes.DWORD, ctypes.POINTER(Blob)]
|
||||
decrypt.restype = wintypes.BOOL
|
||||
if not decrypt(ctypes.byref(source), None, None, None, None, 0, ctypes.byref(result)):
|
||||
raise ctypes.WinError()
|
||||
try:
|
||||
return ctypes.string_at(result.pbData, result.cbData).decode('utf-8')
|
||||
finally:
|
||||
ctypes.windll.kernel32.LocalFree.argtypes = [ctypes.c_void_p]
|
||||
ctypes.windll.kernel32.LocalFree(result.pbData)
|
||||
|
||||
|
||||
def acquire(settings):
|
||||
global _active
|
||||
deadline = time.monotonic() + settings['queueWaitSeconds']
|
||||
with _condition:
|
||||
while settings['maxConcurrent'] and _active >= settings['maxConcurrent']:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
_condition.wait(remaining)
|
||||
_active += 1
|
||||
return True
|
||||
|
||||
|
||||
def release():
|
||||
global _active
|
||||
with _condition:
|
||||
_active -= 1
|
||||
_condition.notify_all()
|
||||
@@ -38,7 +38,7 @@ from bekci_watch import MetadataFilter
|
||||
# Installed only by owned native launcher, never an HTTP control endpoint.
|
||||
WATCH_OWNER = None
|
||||
|
||||
SURUM = "11"
|
||||
SURUM = "12"
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
try:
|
||||
with open(os.path.abspath(__file__), "rb") as _f:
|
||||
@@ -933,12 +933,24 @@ class H(http.server.BaseHTTPRequestHandler):
|
||||
if hata:
|
||||
gecmedi = True
|
||||
elif metin and jev_denetim.muaf_mi(hedef):
|
||||
log(f"{tag} Jev atlandi: Sesli-Hizli-Zeki")
|
||||
log(f"{tag} Jev atlandi: her zaman gec ({hedef})")
|
||||
elif metin:
|
||||
t0 = time.time()
|
||||
puan = jev_denetim.sor(istek, metin, jev_denetim.anahtar())
|
||||
# Jev may wait for a slot or a remote response. Keep this chat alive.
|
||||
import threading as jev_threading
|
||||
jev_done, jev_result = jev_threading.Event(), [None]
|
||||
def denetle():
|
||||
try:
|
||||
jev_result[0] = jev_denetim.sor(istek, metin, jev_denetim.anahtar())
|
||||
finally:
|
||||
jev_done.set()
|
||||
jev_threading.Thread(target=denetle, daemon=True).start()
|
||||
while not jev_done.wait(10):
|
||||
w(b": jev-denetim\n\n")
|
||||
puan = jev_result[0]
|
||||
if puan is None:
|
||||
log(f"{tag} katman denetimi: Jev'e sorulamadi, cevap oldugu gibi geciyor")
|
||||
gecmedi = jev_denetim.hata_gecis()
|
||||
log(f"{tag} katman denetimi: Jev'e sorulamadi, {'siradaki katman' if gecmedi else 'cevap oldugu gibi geciyor'}")
|
||||
else:
|
||||
gecmedi = jev_denetim.karar(puan)
|
||||
log(f"{tag} katman denetimi: ret={puan['ret']:.2f} sart={puan['sart']:.2f} -> "
|
||||
@@ -962,11 +974,16 @@ class H(http.server.BaseHTTPRequestHandler):
|
||||
resp, rewriter, tag = yeni
|
||||
hedef = ad # Muafiyet yalnız ilk exact hedefe ait; yedek katmana taşınmaz.
|
||||
log(f"{tag} katman denetimi: adim bu katmana verildi")
|
||||
if not kalan:
|
||||
if jev_denetim.muaf_mi(hedef):
|
||||
return self._stream(resp, resp.status, tag, rewriter, izle, basliksiz=True)
|
||||
if gecmedi:
|
||||
# Son katman ne dondurduyse o gider, hicbir sey eklenmez; dondurmediyse hicbir sey gitmez.
|
||||
w(son_govde)
|
||||
if son_govde:
|
||||
w(son_govde)
|
||||
else:
|
||||
event = {"type": "response.failed", "response": {"status": "failed", "error": {
|
||||
"code": "jev_no_fallback", "message": "Jev denetiminden gecen cevap alinamadi; gecis kurallarini kontrol edin."}}}
|
||||
w(b"event: response.failed\ndata: " + json.dumps(event).encode() + b"\n\n")
|
||||
else:
|
||||
for chunk in parcalar:
|
||||
if izle is not None:
|
||||
@@ -1064,7 +1081,9 @@ class H(http.server.BaseHTTPRequestHandler):
|
||||
"ide_model": int((sayi.split() or ["0"])[0]),
|
||||
"ozet": bool((CFG.get("ozet_kopru") or {}).get("enabled", False)),
|
||||
"is_hatti": bool(HAT_KOMUT[0]),
|
||||
"adim_denetimi": bool(jev_denetim.anahtar())})
|
||||
"adim_denetimi": jev_denetim.etkin_mi() and bool(jev_denetim.anahtar()),
|
||||
"jev_models": [{"id": m.get("target"), "name": m.get("display_name") or m.get("target")}
|
||||
for m in EXTRA_MODELS if m.get("route") == IDE_ROUTE and m.get("target")]})
|
||||
if self.command == "POST" and path.rstrip("/").endswith("/stop"):
|
||||
self._send_json(200, {"ok": True})
|
||||
log("AITURK IDE istegiyle kapatiliyor")
|
||||
@@ -1720,9 +1739,9 @@ class H(http.server.BaseHTTPRequestHandler):
|
||||
# secili model baska bir rotada olsa bile. Gecit yoksa denetim yapilmaz. Ozet isteginde ve aracsiz
|
||||
# (rewriter yok) yolda kapalidir.
|
||||
jev_rota = ROUTES.get(ide_route)
|
||||
jev_sira = ([] if ozet_tetik or rewriter is None or jev_rota is None or not jev_denetim.anahtar()
|
||||
else jev_denetim.sonrakiler(target))
|
||||
jev_istek = jev_denetim.son_istek(req) if jev_sira else ""
|
||||
jev_etkin = jev_denetim.etkin_mi() and not (ozet_tetik or rewriter is None or jev_rota is None or not jev_denetim.anahtar())
|
||||
jev_sira = jev_denetim.sonrakiler(target) if jev_etkin else []
|
||||
jev_istek = jev_denetim.son_istek(req) if jev_etkin else ""
|
||||
|
||||
def jev_ac(ad):
|
||||
katman_req = json.loads(data)
|
||||
@@ -1758,7 +1777,7 @@ class H(http.server.BaseHTTPRequestHandler):
|
||||
log(f"{tag} deneme {attempt}: HTTP {e.code} {errbody[:100]!r} -> tekrar")
|
||||
time.sleep(min(2 * attempt, 8)); continue
|
||||
log(f"{tag} HTTP {e.code} {errbody[:200]!r}")
|
||||
if jev_sira: # secili model hata verdi: adimi siradaki katman yapar
|
||||
if jev_etkin: # secili model hata verdi: adimi siradaki katman yapar
|
||||
return self._denetimli_akis(None, tag, rewriter, izle, jev_istek, jev_sira, jev_ac, hedef=target)
|
||||
if pencere.tasma_mi(etxt):
|
||||
# Hedef "baglam sigmiyor" dedi: Codex'in tanidigi tasma hatasi olarak iletilir ve hedefin
|
||||
@@ -1805,10 +1824,10 @@ class H(http.server.BaseHTTPRequestHandler):
|
||||
sonuc = self._stream(resp, resp.status, tag, akis)
|
||||
log(f"{tag} ozet kopru: {akis.sonuc_notu()}")
|
||||
return sonuc
|
||||
if jev_sira and resp.status == 200:
|
||||
if jev_etkin and resp.status == 200:
|
||||
return self._denetimli_akis(resp, tag, rewriter, izle, jev_istek, jev_sira, jev_ac, hedef=target)
|
||||
return self._stream(resp, resp.status, tag, rewriter, izle)
|
||||
if jev_sira: # secili modele ulasilamadi: adimi siradaki katman yapar
|
||||
if jev_etkin: # secili modele ulasilamadi: adimi siradaki katman yapar
|
||||
return self._denetimli_akis(None, tag, rewriter, izle, jev_istek, jev_sira, jev_ac, hedef=target)
|
||||
self._send_json(503, {"error": {"message": f"bridge: hedef ulasilamadi {last}"}})
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ class Katman(unittest.TestCase):
|
||||
self.assertEqual(jev_denetim.sonrakiler("Katman-3"), ["Katman-4"])
|
||||
|
||||
def test_last_layer_is_never_asked(self):
|
||||
self.assertEqual(jev_denetim.sonrakiler("Katman-4"), [])
|
||||
self.assertTrue(jev_denetim.muaf_mi("Katman-4"))
|
||||
self.assertEqual(jev_denetim.sonrakiler(None), [])
|
||||
|
||||
|
||||
@@ -129,6 +129,7 @@ class Akis(unittest.TestCase):
|
||||
sahte.komutlar = jev_denetim.komutlar
|
||||
sahte.hazirlik_mi = jev_denetim.hazirlik_mi
|
||||
sahte.muaf_mi = jev_denetim.muaf_mi
|
||||
sahte.hata_gecis = jev_denetim.hata_gecis
|
||||
env = {"jev_denetim": sahte, "yarim": yarim, "time": time, "http": __import__("http.client"),
|
||||
"log": self.logs.append, "RESP_HOP": set(), "json": json}
|
||||
adlar = ("_akis_basligi", "_denetimli_akis", "_stream")
|
||||
@@ -332,7 +333,7 @@ class Akis(unittest.TestCase):
|
||||
self.calistir(sse("Bunu yapamam."))
|
||||
self.assertEqual(self.opened, ["Katman-2", "Katman-3", "Katman-4"])
|
||||
self.assertNotIn(b"yapamam", self.h.wfile.getvalue())
|
||||
self.assertEqual(self.sent(), b"") # the last layer said nothing: nothing is added
|
||||
self.assertIn(b'response.failed', self.sent())
|
||||
|
||||
def test_the_last_layers_own_error_body_is_delivered_untouched(self):
|
||||
class Hata(Exception):
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import jev_denetim
|
||||
import jev_settings
|
||||
|
||||
|
||||
class Settings(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.home = Path(self.tmp.name)
|
||||
self.home_patch = patch.object(jev_settings, 'home', return_value=self.home)
|
||||
self.home_patch.start()
|
||||
self.addCleanup(self.home_patch.stop)
|
||||
self.settings = copy.deepcopy(jev_settings.DEFAULTS)
|
||||
|
||||
def save(self, **changes):
|
||||
self.settings.update(changes)
|
||||
(self.home / 'jev-settings.json').write_text(json.dumps({'settings': self.settings}), encoding='utf-8')
|
||||
|
||||
def test_live_rules_general_override_and_always_pass(self):
|
||||
self.save(fallbackModel='B', rejectionRules=[{'from': 'A', 'to': 'C'}, {'from': 'C', 'to': 'D'}], exemptModels=['D'])
|
||||
self.assertEqual(jev_denetim.sonrakiler('A'), ['C', 'D'])
|
||||
self.assertEqual(jev_denetim.sonrakiler('Other'), ['B'])
|
||||
self.assertTrue(jev_denetim.muaf_mi('D'))
|
||||
self.assertFalse(jev_denetim.muaf_mi('A'))
|
||||
self.save(enabled=False)
|
||||
self.assertFalse(jev_denetim.etkin_mi())
|
||||
self.assertEqual(jev_denetim.sonrakiler('A'), [])
|
||||
|
||||
def test_cycle_cannot_spin_even_if_file_manually_edited(self):
|
||||
self.save(rejectionRules=[{'from': 'A', 'to': 'B'}, {'from': 'B', 'to': 'A'}])
|
||||
self.assertEqual(jev_denetim.sonrakiler('A'), ['B'])
|
||||
|
||||
def test_prompts_model_criteria_and_thresholds_are_applied(self):
|
||||
self.save(model='jev-custom', questions={'ret': 'question A', 'sart': 'question B'}, thresholds={'ret': .9, 'sart': .8}, criteria={'ret': {'true': 'bad', 'false': 'good'}, 'sart': {'true': '', 'false': ''}})
|
||||
captured = []
|
||||
def send(body, key):
|
||||
captured.append(body)
|
||||
return {'answers': {'ret': {'noul': .6}, 'sart': {'noul': .2}}}
|
||||
result = jev_denetim.sor('request', 'answer', 'test', gonder=send)
|
||||
self.assertFalse(jev_denetim.karar(result))
|
||||
self.assertEqual(captured[0]['model'], 'jev-custom')
|
||||
self.assertEqual(captured[0]['questions']['ret']['instructions'], 'question A')
|
||||
self.assertEqual(captured[0]['questions']['ret']['criteria'], {'true': 'bad', 'false': 'good'})
|
||||
self.assertNotIn('criteria', captured[0]['questions']['sart'])
|
||||
self.save(thresholds={'ret': .1, 'sart': .1})
|
||||
self.assertFalse(jev_denetim.karar(result)) # in-flight result retains its own thresholds
|
||||
|
||||
def test_concurrency_is_limited_without_serializing_all_requests(self):
|
||||
self.save(maxConcurrent=2)
|
||||
count, peak = 0, 0
|
||||
lock = threading.Lock()
|
||||
def send(body, key):
|
||||
nonlocal count, peak
|
||||
with lock:
|
||||
count += 1
|
||||
peak = max(peak, count)
|
||||
time.sleep(.04)
|
||||
with lock:
|
||||
count -= 1
|
||||
return {'answers': {'ret': {'noul': 0}, 'sart': {'noul': 0}}}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
answers = list(pool.map(lambda _: jev_denetim.sor('r', 'a', 'test', gonder=send), range(8)))
|
||||
self.assertEqual(peak, 2)
|
||||
self.assertTrue(all(a is not None for a in answers))
|
||||
|
||||
def test_timeout_endpoint_and_failure_behavior(self):
|
||||
self.save(endpoint='https://example.test/jev', timeoutSeconds=42, failureAction='next')
|
||||
with patch('urllib.request.urlopen', side_effect=TimeoutError) as opening:
|
||||
self.assertIsNone(jev_denetim.sor('r', 'a', 'test'))
|
||||
self.assertEqual(opening.call_args.kwargs['timeout'], 42)
|
||||
self.assertEqual(opening.call_args.args[0].full_url, 'https://example.test/jev')
|
||||
self.assertTrue(jev_denetim.hata_gecis())
|
||||
self.assertEqual(jev_settings._active, 0)
|
||||
|
||||
def test_queue_wait_expires_and_slot_count_is_preserved(self):
|
||||
self.save(maxConcurrent=1, queueWaitSeconds=1)
|
||||
settings = jev_settings.read()[0]
|
||||
self.assertTrue(jev_settings.acquire(settings))
|
||||
try:
|
||||
with patch.object(jev_denetim, '_gonder') as send:
|
||||
self.assertIsNone(jev_denetim.sor('r', 'a', 'test', gonder=send))
|
||||
send.assert_not_called()
|
||||
self.assertEqual(jev_settings._active, 1)
|
||||
finally:
|
||||
jev_settings.release()
|
||||
|
||||
@unittest.skipUnless(os.name == 'nt', 'Windows DPAPI')
|
||||
def test_encrypted_key_roundtrip_without_plaintext_on_disk(self):
|
||||
secret = 'dummy-key-for-local-test'
|
||||
code = 'Add-Type -AssemblyName System.Security; $b=[Text.Encoding]::UTF8.GetBytes([Console]::In.ReadToEnd()); [Console]::Out.Write([Convert]::ToBase64String([Security.Cryptography.ProtectedData]::Protect($b,$null,[Security.Cryptography.DataProtectionScope]::CurrentUser)))'
|
||||
encrypted = subprocess.run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command', code], input=secret, capture_output=True, text=True, check=True, creationflags=subprocess.CREATE_NO_WINDOW).stdout.strip()
|
||||
text = json.dumps({'settings': self.settings, 'protectedKey': encrypted})
|
||||
(self.home / 'jev-settings.json').write_text(text, encoding='utf-8')
|
||||
self.assertNotIn(secret, text)
|
||||
self.assertEqual(jev_settings.key(), secret)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user