feat(settings): configure Jev evaluation and model transitions

This commit is contained in:
yilsem committed 2026-10-13 01:07:58 +03:00
1 parent 369f6c2ca8
commit 883b1dc824
20 files changed
+1195 -37

No files matched your search

+1 -1
View File
@@ -34,7 +34,7 @@ import urllib.request
import gecis_context
HERE = os.path.dirname(os.path.abspath(__file__))
SURUM = "11" # AITURK IDE'nin bekledigi kopru surumu (codex-link-config.ts CODEX_BRIDGE_VERSION)
SURUM = "12" # AITURK IDE'nin bekledigi kopru surumu (codex-link-config.ts CODEX_BRIDGE_VERSION)
try:
with open(os.path.abspath(__file__), "rb") as _f:
KOD = hashlib.sha256(_f.read()).hexdigest()[:8]
@@ -0,0 +1,22 @@
{
"enabled": true,
"endpoint": "https://api.typesafe.ai/v1/systemone",
"model": "jev-1.13.0",
"fallbackModel": "Katman-1",
"rejectionRules": [
{ "from": "Katman-1", "to": "Katman-2" },
{ "from": "Katman-2", "to": "Katman-3" },
{ "from": "Katman-3", "to": "Katman-4" }
],
"exemptModels": ["Sesli-Hizli-Zeki", "Katman-4"],
"thresholds": { "ret": 0.5, "sart": 0.7 },
"questions": {
"ret": "The assistant did NOT do what the user asked or did NOT answer the user's question: it refused, declined, said it cannot or will not, avoided giving the requested judgment, or redirected to a different topic.",
"sart": "The assistant changed the request on its own: it added its own conditions, softened or relabeled what was asked, or left part of it out, instead of doing exactly what the user asked."
},
"timeoutSeconds": 3600,
"criteria": { "ret": { "true": "", "false": "" }, "sart": { "true": "", "false": "" } },
"maxConcurrent": 0,
"queueWaitSeconds": 3600,
"failureAction": "pass"
}
+64 -16
View File
@@ -18,6 +18,7 @@ import json
import os
import re
import urllib.request
import jev_settings
ADRES = "https://api.typesafe.ai/v1/systemone"
MODEL = "jev-1.13.0"
@@ -32,26 +33,44 @@ SORU = {
_ANAHTAR = {"okundu": False, "deger": ""}
def muaf_mi(hedef):
"""12 Ekim 2026: yalnız çözülmüş exact sesli combo hedefi Jev'den muaftır.
"""Settings' always-pass list matches exact resolved model IDs.
Taşıma hatalarının katman sırasını değiştirmez; yedek model kendi adına denetlenir.
"""
return hedef == "Sesli-Hizli-Zeki"
return hedef in jev_settings.read()[0]['exemptModels']
def sonrakiler(hedef):
"""Jev gecirmezse sirayla denenecek katmanlar. Secili model bir katmansa ondan sonrakiler; katman degilse
butun katmanlar bastan (once Katman-1). Son katman (Katman-4) kendisi gelince bos doner: sorgulanmadan gecer."""
"""Model-specific rejection edges take precedence over the general fallback.
Stop at an always-pass model or before revisiting any target.
"""
settings = jev_settings.read()[0]
ad = str(hedef or "")
if not ad or ad == KATMAN[-1]:
if not settings['enabled'] or not ad:
return []
if ad in KATMAN:
return list(KATMAN[KATMAN.index(ad) + 1:])
return list(KATMAN)
routes = {r['from']: r['to'] for r in settings['rejectionRules']}
result, seen = [], {ad}
# An exempt model can still use the general fallback on transport failure.
# Repeated targets stop the walk even if a file was edited outside Settings.
while len(result) < 65:
target = routes.get(ad, settings['fallbackModel'])
if target in seen:
break
seen.add(target)
result.append(target)
if target in settings['exemptModels']:
break
ad = target
return result
def anahtar(ortam=None, dosyalar=None):
"""TypeSafe anahtari: ortam degiskeni, sonra ~/.typesafe/typesafe.env. Yoksa ""."""
if ortam is None and dosyalar is None:
saved = jev_settings.key()
if saved:
return saved
ortam = os.environ if ortam is None else ortam
deger = str(ortam.get("TYPESAFE_API_KEY") or "").strip()
if deger:
@@ -147,24 +166,53 @@ def komutlar(govde):
return "\n".join(satirlar)
def _gonder(govde, key):
istek = urllib.request.Request(ADRES, data=json.dumps(govde).encode("utf-8"), method="POST",
def _gonder(govde, key, settings=None):
settings = settings or jev_settings.read()[0]
istek = urllib.request.Request(settings['endpoint'], data=json.dumps(govde).encode("utf-8"), method="POST",
headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"})
with urllib.request.urlopen(istek) as yanit:
with urllib.request.urlopen(istek, timeout=settings['timeoutSeconds']) as yanit:
return json.load(yanit)
class Puan(dict):
"""Carry the thresholds used for this evaluation through a settings change."""
pass
def sor(istek, cevap, key, gonder=_gonder):
"""Jev'e sorar. Donus: {"ret", "sart"} olasiliklari | None (sorulamadi)."""
govde = {"model": MODEL, "state": {"user_request": istek, "assistant_reply": cevap},
"questions": {k: {"type": "noul", "instructions": v} for k, v in SORU.items()}}
settings = jev_settings.read()[0]
govde = {"model": settings['model'], "state": {"user_request": istek, "assistant_reply": cevap},
"questions": {k: {"type": "noul", "instructions": v} for k, v in settings['questions'].items()}}
for name, criteria in settings.get('criteria', {}).items():
values = {k: v for k, v in criteria.items() if v.strip()}
if values:
govde['questions'][name]['criteria'] = values
if not jev_settings.acquire(settings):
return None
try:
yanitlar = (gonder(govde, key) or {}).get("answers") or {}
return {k: float(yanitlar[k]["noul"]) for k in SORU}
response = gonder(govde, key, settings) if gonder is _gonder else gonder(govde, key)
yanitlar = (response or {}).get("answers") or {}
result = Puan({k: float(yanitlar[k]["noul"]) for k in SORU})
if any(not 0 <= v <= 1 for v in result.values()):
return None
result.thresholds = settings['thresholds']
return result
except Exception:
return None
finally:
jev_settings.release()
def karar(puan):
"""True: adim gecmez, sonraki katman yapar. Istisna yok."""
return puan["ret"] >= ESIK["ret"] or puan["sart"] >= ESIK["sart"]
thresholds = getattr(puan, 'thresholds', None) or jev_settings.read()[0]['thresholds']
return puan["ret"] >= thresholds["ret"] or puan["sart"] >= thresholds["sart"]
def hata_gecis():
return jev_settings.read()[0]['failureAction'] == 'next'
def etkin_mi():
return bool(jev_settings.read()[0]['enabled'])
@@ -0,0 +1,89 @@
"""Per-user settings, re-read for new evaluations without restarting active chats."""
import base64
import copy
import ctypes
import json
import os
import sys
import threading
import time
from pathlib import Path
DEFAULTS = json.loads(Path(__file__).with_name('jev-settings-defaults.json').read_text(encoding='utf-8'))
_condition = threading.Condition()
_active = 0
def home():
if '--home' in sys.argv:
return Path(sys.argv[sys.argv.index('--home') + 1])
return Path(os.environ.get('AITURK_CODEX_BRIDGE_HOME') or Path(__file__).parent)
def read():
try:
data = json.loads((home() / 'jev-settings.json').read_text(encoding='utf-8'))
except FileNotFoundError:
return copy.deepcopy(DEFAULTS), None
settings = data['settings']
# A corrupted file must not silently change routing or contact another host.
from urllib.parse import urlsplit
url = urlsplit(settings['endpoint'])
if url.scheme != 'https' or not url.netloc or url.username or url.password:
raise ValueError('Invalid Jev endpoint')
for name in ('ret', 'sart'):
n = settings['thresholds'][name]
if not isinstance(n, (int, float)) or not 0 <= n <= 1 or not settings['questions'][name]:
raise ValueError('Invalid Jev rule')
for name in ('timeoutSeconds', 'queueWaitSeconds'):
if not isinstance(settings[name], int) or not 1 <= settings[name] <= 3600:
raise ValueError('Invalid Jev timeout')
if not isinstance(settings['maxConcurrent'], int) or not 0 <= settings['maxConcurrent'] <= 512:
raise ValueError('Invalid Jev concurrency')
if not settings['fallbackModel'] or not isinstance(settings['rejectionRules'], list) or settings['failureAction'] not in ('pass', 'next'):
raise ValueError('Invalid Jev routing')
return settings, data.get('protectedKey')
def key():
_, encrypted = read()
if not encrypted:
return ''
if os.name != 'nt':
raise RuntimeError('Windows credential protection required')
from ctypes import wintypes
class Blob(ctypes.Structure):
_fields_ = [('cbData', wintypes.DWORD), ('pbData', ctypes.POINTER(ctypes.c_ubyte))]
raw = base64.b64decode(encrypted, validate=True)
buf = (ctypes.c_ubyte * len(raw)).from_buffer_copy(raw)
source, result = Blob(len(raw), buf), Blob()
decrypt = ctypes.windll.crypt32.CryptUnprotectData
decrypt.argtypes = [ctypes.POINTER(Blob), ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, wintypes.DWORD, ctypes.POINTER(Blob)]
decrypt.restype = wintypes.BOOL
if not decrypt(ctypes.byref(source), None, None, None, None, 0, ctypes.byref(result)):
raise ctypes.WinError()
try:
return ctypes.string_at(result.pbData, result.cbData).decode('utf-8')
finally:
ctypes.windll.kernel32.LocalFree.argtypes = [ctypes.c_void_p]
ctypes.windll.kernel32.LocalFree(result.pbData)
def acquire(settings):
global _active
deadline = time.monotonic() + settings['queueWaitSeconds']
with _condition:
while settings['maxConcurrent'] and _active >= settings['maxConcurrent']:
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
_condition.wait(remaining)
_active += 1
return True
def release():
global _active
with _condition:
_active -= 1
_condition.notify_all()
+32 -13
View File
@@ -38,7 +38,7 @@ from bekci_watch import MetadataFilter
# Installed only by owned native launcher, never an HTTP control endpoint.
WATCH_OWNER = None
SURUM = "11"
SURUM = "12"
HERE = os.path.dirname(os.path.abspath(__file__))
try:
with open(os.path.abspath(__file__), "rb") as _f:
@@ -933,12 +933,24 @@ class H(http.server.BaseHTTPRequestHandler):
if hata:
gecmedi = True
elif metin and jev_denetim.muaf_mi(hedef):
log(f"{tag} Jev atlandi: Sesli-Hizli-Zeki")
log(f"{tag} Jev atlandi: her zaman gec ({hedef})")
elif metin:
t0 = time.time()
puan = jev_denetim.sor(istek, metin, jev_denetim.anahtar())
# Jev may wait for a slot or a remote response. Keep this chat alive.
import threading as jev_threading
jev_done, jev_result = jev_threading.Event(), [None]
def denetle():
try:
jev_result[0] = jev_denetim.sor(istek, metin, jev_denetim.anahtar())
finally:
jev_done.set()
jev_threading.Thread(target=denetle, daemon=True).start()
while not jev_done.wait(10):
w(b": jev-denetim\n\n")
puan = jev_result[0]
if puan is None:
log(f"{tag} katman denetimi: Jev'e sorulamadi, cevap oldugu gibi geciyor")
gecmedi = jev_denetim.hata_gecis()
log(f"{tag} katman denetimi: Jev'e sorulamadi, {'siradaki katman' if gecmedi else 'cevap oldugu gibi geciyor'}")
else:
gecmedi = jev_denetim.karar(puan)
log(f"{tag} katman denetimi: ret={puan['ret']:.2f} sart={puan['sart']:.2f} -> "
@@ -962,11 +974,16 @@ class H(http.server.BaseHTTPRequestHandler):
resp, rewriter, tag = yeni
hedef = ad # Muafiyet yalnız ilk exact hedefe ait; yedek katmana taşınmaz.
log(f"{tag} katman denetimi: adim bu katmana verildi")
if not kalan:
if jev_denetim.muaf_mi(hedef):
return self._stream(resp, resp.status, tag, rewriter, izle, basliksiz=True)
if gecmedi:
# Son katman ne dondurduyse o gider, hicbir sey eklenmez; dondurmediyse hicbir sey gitmez.
w(son_govde)
if son_govde:
w(son_govde)
else:
event = {"type": "response.failed", "response": {"status": "failed", "error": {
"code": "jev_no_fallback", "message": "Jev denetiminden gecen cevap alinamadi; gecis kurallarini kontrol edin."}}}
w(b"event: response.failed\ndata: " + json.dumps(event).encode() + b"\n\n")
else:
for chunk in parcalar:
if izle is not None:
@@ -1064,7 +1081,9 @@ class H(http.server.BaseHTTPRequestHandler):
"ide_model": int((sayi.split() or ["0"])[0]),
"ozet": bool((CFG.get("ozet_kopru") or {}).get("enabled", False)),
"is_hatti": bool(HAT_KOMUT[0]),
"adim_denetimi": bool(jev_denetim.anahtar())})
"adim_denetimi": jev_denetim.etkin_mi() and bool(jev_denetim.anahtar()),
"jev_models": [{"id": m.get("target"), "name": m.get("display_name") or m.get("target")}
for m in EXTRA_MODELS if m.get("route") == IDE_ROUTE and m.get("target")]})
if self.command == "POST" and path.rstrip("/").endswith("/stop"):
self._send_json(200, {"ok": True})
log("AITURK IDE istegiyle kapatiliyor")
@@ -1720,9 +1739,9 @@ class H(http.server.BaseHTTPRequestHandler):
# secili model baska bir rotada olsa bile. Gecit yoksa denetim yapilmaz. Ozet isteginde ve aracsiz
# (rewriter yok) yolda kapalidir.
jev_rota = ROUTES.get(ide_route)
jev_sira = ([] if ozet_tetik or rewriter is None or jev_rota is None or not jev_denetim.anahtar()
else jev_denetim.sonrakiler(target))
jev_istek = jev_denetim.son_istek(req) if jev_sira else ""
jev_etkin = jev_denetim.etkin_mi() and not (ozet_tetik or rewriter is None or jev_rota is None or not jev_denetim.anahtar())
jev_sira = jev_denetim.sonrakiler(target) if jev_etkin else []
jev_istek = jev_denetim.son_istek(req) if jev_etkin else ""
def jev_ac(ad):
katman_req = json.loads(data)
@@ -1758,7 +1777,7 @@ class H(http.server.BaseHTTPRequestHandler):
log(f"{tag} deneme {attempt}: HTTP {e.code} {errbody[:100]!r} -> tekrar")
time.sleep(min(2 * attempt, 8)); continue
log(f"{tag} HTTP {e.code} {errbody[:200]!r}")
if jev_sira: # secili model hata verdi: adimi siradaki katman yapar
if jev_etkin: # secili model hata verdi: adimi siradaki katman yapar
return self._denetimli_akis(None, tag, rewriter, izle, jev_istek, jev_sira, jev_ac, hedef=target)
if pencere.tasma_mi(etxt):
# Hedef "baglam sigmiyor" dedi: Codex'in tanidigi tasma hatasi olarak iletilir ve hedefin
@@ -1805,10 +1824,10 @@ class H(http.server.BaseHTTPRequestHandler):
sonuc = self._stream(resp, resp.status, tag, akis)
log(f"{tag} ozet kopru: {akis.sonuc_notu()}")
return sonuc
if jev_sira and resp.status == 200:
if jev_etkin and resp.status == 200:
return self._denetimli_akis(resp, tag, rewriter, izle, jev_istek, jev_sira, jev_ac, hedef=target)
return self._stream(resp, resp.status, tag, rewriter, izle)
if jev_sira: # secili modele ulasilamadi: adimi siradaki katman yapar
if jev_etkin: # secili modele ulasilamadi: adimi siradaki katman yapar
return self._denetimli_akis(None, tag, rewriter, izle, jev_istek, jev_sira, jev_ac, hedef=target)
self._send_json(503, {"error": {"message": f"bridge: hedef ulasilamadi {last}"}})
@@ -71,7 +71,7 @@ class Katman(unittest.TestCase):
self.assertEqual(jev_denetim.sonrakiler("Katman-3"), ["Katman-4"])
def test_last_layer_is_never_asked(self):
self.assertEqual(jev_denetim.sonrakiler("Katman-4"), [])
self.assertTrue(jev_denetim.muaf_mi("Katman-4"))
self.assertEqual(jev_denetim.sonrakiler(None), [])
@@ -129,6 +129,7 @@ class Akis(unittest.TestCase):
sahte.komutlar = jev_denetim.komutlar
sahte.hazirlik_mi = jev_denetim.hazirlik_mi
sahte.muaf_mi = jev_denetim.muaf_mi
sahte.hata_gecis = jev_denetim.hata_gecis
env = {"jev_denetim": sahte, "yarim": yarim, "time": time, "http": __import__("http.client"),
"log": self.logs.append, "RESP_HOP": set(), "json": json}
adlar = ("_akis_basligi", "_denetimli_akis", "_stream")
@@ -332,7 +333,7 @@ class Akis(unittest.TestCase):
self.calistir(sse("Bunu yapamam."))
self.assertEqual(self.opened, ["Katman-2", "Katman-3", "Katman-4"])
self.assertNotIn(b"yapamam", self.h.wfile.getvalue())
self.assertEqual(self.sent(), b"") # the last layer said nothing: nothing is added
self.assertIn(b'response.failed', self.sent())
def test_the_last_layers_own_error_body_is_delivered_untouched(self):
class Hata(Exception):
@@ -0,0 +1,111 @@
import copy
import json
import os
import subprocess
import tempfile
import threading
import time
import unittest
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from unittest.mock import patch
import jev_denetim
import jev_settings
class Settings(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.home = Path(self.tmp.name)
self.home_patch = patch.object(jev_settings, 'home', return_value=self.home)
self.home_patch.start()
self.addCleanup(self.home_patch.stop)
self.settings = copy.deepcopy(jev_settings.DEFAULTS)
def save(self, **changes):
self.settings.update(changes)
(self.home / 'jev-settings.json').write_text(json.dumps({'settings': self.settings}), encoding='utf-8')
def test_live_rules_general_override_and_always_pass(self):
self.save(fallbackModel='B', rejectionRules=[{'from': 'A', 'to': 'C'}, {'from': 'C', 'to': 'D'}], exemptModels=['D'])
self.assertEqual(jev_denetim.sonrakiler('A'), ['C', 'D'])
self.assertEqual(jev_denetim.sonrakiler('Other'), ['B'])
self.assertTrue(jev_denetim.muaf_mi('D'))
self.assertFalse(jev_denetim.muaf_mi('A'))
self.save(enabled=False)
self.assertFalse(jev_denetim.etkin_mi())
self.assertEqual(jev_denetim.sonrakiler('A'), [])
def test_cycle_cannot_spin_even_if_file_manually_edited(self):
self.save(rejectionRules=[{'from': 'A', 'to': 'B'}, {'from': 'B', 'to': 'A'}])
self.assertEqual(jev_denetim.sonrakiler('A'), ['B'])
def test_prompts_model_criteria_and_thresholds_are_applied(self):
self.save(model='jev-custom', questions={'ret': 'question A', 'sart': 'question B'}, thresholds={'ret': .9, 'sart': .8}, criteria={'ret': {'true': 'bad', 'false': 'good'}, 'sart': {'true': '', 'false': ''}})
captured = []
def send(body, key):
captured.append(body)
return {'answers': {'ret': {'noul': .6}, 'sart': {'noul': .2}}}
result = jev_denetim.sor('request', 'answer', 'test', gonder=send)
self.assertFalse(jev_denetim.karar(result))
self.assertEqual(captured[0]['model'], 'jev-custom')
self.assertEqual(captured[0]['questions']['ret']['instructions'], 'question A')
self.assertEqual(captured[0]['questions']['ret']['criteria'], {'true': 'bad', 'false': 'good'})
self.assertNotIn('criteria', captured[0]['questions']['sart'])
self.save(thresholds={'ret': .1, 'sart': .1})
self.assertFalse(jev_denetim.karar(result)) # in-flight result retains its own thresholds
def test_concurrency_is_limited_without_serializing_all_requests(self):
self.save(maxConcurrent=2)
count, peak = 0, 0
lock = threading.Lock()
def send(body, key):
nonlocal count, peak
with lock:
count += 1
peak = max(peak, count)
time.sleep(.04)
with lock:
count -= 1
return {'answers': {'ret': {'noul': 0}, 'sart': {'noul': 0}}}
with ThreadPoolExecutor(max_workers=8) as pool:
answers = list(pool.map(lambda _: jev_denetim.sor('r', 'a', 'test', gonder=send), range(8)))
self.assertEqual(peak, 2)
self.assertTrue(all(a is not None for a in answers))
def test_timeout_endpoint_and_failure_behavior(self):
self.save(endpoint='https://example.test/jev', timeoutSeconds=42, failureAction='next')
with patch('urllib.request.urlopen', side_effect=TimeoutError) as opening:
self.assertIsNone(jev_denetim.sor('r', 'a', 'test'))
self.assertEqual(opening.call_args.kwargs['timeout'], 42)
self.assertEqual(opening.call_args.args[0].full_url, 'https://example.test/jev')
self.assertTrue(jev_denetim.hata_gecis())
self.assertEqual(jev_settings._active, 0)
def test_queue_wait_expires_and_slot_count_is_preserved(self):
self.save(maxConcurrent=1, queueWaitSeconds=1)
settings = jev_settings.read()[0]
self.assertTrue(jev_settings.acquire(settings))
try:
with patch.object(jev_denetim, '_gonder') as send:
self.assertIsNone(jev_denetim.sor('r', 'a', 'test', gonder=send))
send.assert_not_called()
self.assertEqual(jev_settings._active, 1)
finally:
jev_settings.release()
@unittest.skipUnless(os.name == 'nt', 'Windows DPAPI')
def test_encrypted_key_roundtrip_without_plaintext_on_disk(self):
secret = 'dummy-key-for-local-test'
code = 'Add-Type -AssemblyName System.Security; $b=[Text.Encoding]::UTF8.GetBytes([Console]::In.ReadToEnd()); [Console]::Out.Write([Convert]::ToBase64String([Security.Cryptography.ProtectedData]::Protect($b,$null,[Security.Cryptography.DataProtectionScope]::CurrentUser)))'
encrypted = subprocess.run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command', code], input=secret, capture_output=True, text=True, check=True, creationflags=subprocess.CREATE_NO_WINDOW).stdout.strip()
text = json.dumps({'settings': self.settings, 'protectedKey': encrypted})
(self.home / 'jev-settings.json').write_text(text, encoding='utf-8')
self.assertNotIn(secret, text)
self.assertEqual(jev_settings.key(), secret)
if __name__ == '__main__':
unittest.main()