358 lines
16 KiB
Python
358 lines
16 KiB
Python
"""Layer check (11 Oct 2026): Jev passes or blocks each step of Katman-1..3; a blocked step is redone by the next
|
||
layer. Katman-4 is never asked. No exceptions."""
|
||
import ast
|
||
import io
|
||
import json
|
||
import os
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from unittest.mock import Mock
|
||
|
||
import jev_denetim
|
||
import yarim
|
||
|
||
SOURCE = Path(__file__).with_name("kopru.py").read_text(encoding="utf-8")
|
||
tree = ast.parse(SOURCE)
|
||
handler = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "H")
|
||
|
||
|
||
def method(name):
|
||
return next(n for n in handler.body if isinstance(n, ast.FunctionDef) and n.name == name)
|
||
|
||
|
||
def user(text):
|
||
return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}
|
||
|
||
|
||
def sse(*deltas):
|
||
out = b""
|
||
for d in deltas:
|
||
out += b"event: response.output_text.delta\ndata: " + json.dumps(
|
||
{"type": "response.output_text.delta", "delta": d}).encode() + b"\n\n"
|
||
return out + b'event: response.completed\ndata: {"type": "response.completed"}\n\n'
|
||
|
||
|
||
class Yanit:
|
||
status = 200
|
||
|
||
def __init__(self, body):
|
||
self.parts = [body[:len(body) // 2], body[len(body) // 2:]]
|
||
self.headers = {"Content-Type": "text/event-stream"}
|
||
|
||
def read1(self, n):
|
||
return self.parts.pop(0) if self.parts else b""
|
||
|
||
|
||
class Karar(unittest.TestCase):
|
||
def test_refusal_or_own_conditions_block(self):
|
||
self.assertTrue(jev_denetim.karar({"ret": 0.84, "sart": 0.1}))
|
||
self.assertTrue(jev_denetim.karar({"ret": 0.41, "sart": 0.95}))
|
||
self.assertTrue(jev_denetim.karar({"ret": 0.5, "sart": 0.0}))
|
||
|
||
def test_normal_answer_passes(self):
|
||
self.assertFalse(jev_denetim.karar({"ret": 0.05, "sart": 0.3}))
|
||
|
||
def test_no_exception_questions_exist(self):
|
||
self.assertEqual(set(jev_denetim.SORU), {"ret", "sart"})
|
||
|
||
|
||
class Katman(unittest.TestCase):
|
||
def test_a_non_layer_model_falls_through_every_layer_from_the_top(self):
|
||
self.assertEqual(jev_denetim.sonrakiler("Uncensored-Uygulama"),
|
||
["Katman-1", "Katman-2", "Katman-3", "Katman-4"])
|
||
self.assertEqual(jev_denetim.sonrakiler("ide/some-model"),
|
||
["Katman-1", "Katman-2", "Katman-3", "Katman-4"])
|
||
|
||
def test_a_selected_layer_only_tries_the_ones_after_it(self):
|
||
self.assertEqual(jev_denetim.sonrakiler("Katman-1"), ["Katman-2", "Katman-3", "Katman-4"])
|
||
self.assertEqual(jev_denetim.sonrakiler("Katman-2"), ["Katman-3", "Katman-4"])
|
||
self.assertEqual(jev_denetim.sonrakiler("Katman-3"), ["Katman-4"])
|
||
|
||
def test_last_layer_is_never_asked(self):
|
||
self.assertEqual(jev_denetim.sonrakiler("Katman-4"), [])
|
||
self.assertEqual(jev_denetim.sonrakiler(None), [])
|
||
|
||
|
||
class Istek(unittest.TestCase):
|
||
def test_last_user_request_is_taken_and_wrappers_are_skipped(self):
|
||
voice = user("<realtime_delegation>\n <input>siteyi yap</input>\n</realtime_delegation>")
|
||
self.assertEqual(jev_denetim.son_istek({"input": [user("eski"), voice]}), "siteyi yap")
|
||
hook = user("<hook_context>pano</hook_context>")
|
||
call = {"type": "function_call", "call_id": "c", "name": "shell", "arguments": "{}"}
|
||
self.assertEqual(jev_denetim.son_istek({"input": [user("README'yi düzelt"), call, hook]}), "README'yi düzelt")
|
||
self.assertEqual(jev_denetim.son_istek({"input": [call]}), "")
|
||
|
||
|
||
class Sor(unittest.TestCase):
|
||
def test_request_shape_and_scores(self):
|
||
seen = {}
|
||
|
||
def gonder(govde, key):
|
||
seen.update(govde=govde, key=key)
|
||
return {"answers": {k: {"type": "noul", "noul": 0.25} for k in jev_denetim.SORU}}
|
||
self.assertEqual(jev_denetim.sor("istek", "c" * 9000, "anahtar", gonder=gonder), {"ret": 0.25, "sart": 0.25})
|
||
self.assertEqual(seen["key"], "anahtar")
|
||
self.assertEqual(seen["govde"]["state"]["user_request"], "istek")
|
||
self.assertEqual(seen["govde"]["state"]["assistant_reply"], "c" * 9000)
|
||
|
||
def test_failures_return_none_every_time(self):
|
||
calls = []
|
||
|
||
def bozuk(govde, key):
|
||
calls.append(1)
|
||
raise OSError("ulasilamadi")
|
||
for _ in range(5):
|
||
self.assertIsNone(jev_denetim.sor("i", "c", "k", gonder=bozuk))
|
||
self.assertEqual(len(calls), 5)
|
||
|
||
def test_key_is_read_from_env_or_file(self):
|
||
self.assertEqual(jev_denetim.anahtar({"TYPESAFE_API_KEY": " a "}), "a")
|
||
with tempfile.TemporaryDirectory() as d:
|
||
yol = os.path.join(d, "typesafe.env")
|
||
Path(yol).write_text('TYPESAFE_API_KEY="b"\n', encoding="utf-8")
|
||
self.assertEqual(jev_denetim.anahtar({}, [yol]), "b")
|
||
self.assertEqual(jev_denetim.anahtar({}, [os.path.join(d, "yok.env")]), "")
|
||
|
||
|
||
class Akis(unittest.TestCase):
|
||
"""_denetimli_akis through a fake handler: what reaches the client."""
|
||
|
||
def setUp(self):
|
||
self.logs, self.asked, self.opened = [], [], []
|
||
self.puanlar = [] # one verdict per asked answer; empty -> pass
|
||
sahte = Mock()
|
||
sahte.anahtar = lambda: "k"
|
||
sahte.sor = self.sor
|
||
sahte.karar = jev_denetim.karar
|
||
sahte.komutlar = jev_denetim.komutlar
|
||
sahte.hazirlik_mi = jev_denetim.hazirlik_mi
|
||
sahte.muaf_mi = jev_denetim.muaf_mi
|
||
env = {"jev_denetim": sahte, "yarim": yarim, "time": time, "http": __import__("http.client"),
|
||
"log": self.logs.append, "RESP_HOP": set(), "json": json}
|
||
adlar = ("_akis_basligi", "_denetimli_akis", "_stream")
|
||
exec(compile(ast.Module(body=[method(a) for a in adlar], type_ignores=[]), "kopru.py", "exec"), env)
|
||
self.h = type("H", (), {a: env[a] for a in adlar})()
|
||
self.h.wfile = io.BytesIO()
|
||
self.h.send_response = Mock()
|
||
self.h.send_header = Mock()
|
||
self.h.end_headers = Mock()
|
||
self.cevap = {"Katman-1": sse("bir"), "Katman-2": sse("iki"), "Katman-3": sse("uc"), "Katman-4": sse("dort")}
|
||
|
||
def sor(self, istek, metin, key):
|
||
self.asked.append(metin)
|
||
return self.puanlar.pop(0) if self.puanlar else {"ret": 0.0, "sart": 0.0}
|
||
|
||
def ac(self, ad):
|
||
self.opened.append(ad)
|
||
if self.cevap[ad] is None:
|
||
raise OSError("acilamadi")
|
||
return Yanit(self.cevap[ad]), None, f"[{ad}]"
|
||
|
||
def sent(self):
|
||
raw, out = self.h.wfile.getvalue(), b""
|
||
while raw:
|
||
size, _, rest = raw.partition(b"\r\n")
|
||
n = int(size, 16)
|
||
if n == 0:
|
||
break
|
||
out += rest[:n]
|
||
raw = rest[n + 2:]
|
||
return out
|
||
|
||
def calistir(self, body, sira=("Katman-2", "Katman-3", "Katman-4")):
|
||
self.h._denetimli_akis(Yanit(body), "[1]", None, None, "yap", list(sira), self.ac)
|
||
|
||
def test_a_passed_step_is_delivered_unchanged(self):
|
||
body = sse("Fenerbahçe'nin ", "13 puanı var.")
|
||
self.calistir(body)
|
||
self.assertEqual(self.sent(), body)
|
||
self.assertEqual(self.asked, ["Fenerbahçe'nin 13 puanı var."])
|
||
self.assertEqual(self.opened, [])
|
||
|
||
def test_exact_voice_target_skips_jev_but_not_transport_fallback(self):
|
||
body = sse("Bunu yapamam.")
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.h._denetimli_akis(Yanit(body), "[1]", None, None, "yap",
|
||
list(jev_denetim.KATMAN), self.ac, hedef="Sesli-Hizli-Zeki")
|
||
self.assertEqual(self.sent(), body)
|
||
self.assertEqual(self.asked, [])
|
||
self.assertEqual(self.opened, [])
|
||
|
||
def test_voice_transport_failure_still_checks_the_fallback_model(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.h._denetimli_akis(None, "[1]", None, None, "yap",
|
||
list(jev_denetim.KATMAN), self.ac, hedef="Sesli-Hizli-Zeki")
|
||
self.assertEqual(self.opened, ["Katman-1", "Katman-2"])
|
||
self.assertEqual(self.asked, ["bir", "iki"])
|
||
self.assertEqual(self.sent(), self.cevap["Katman-2"])
|
||
|
||
def test_voice_lookalikes_are_still_checked(self):
|
||
for target in ("sesli-hizli-zeki", "Sesli-Hizli-Zeki-v2", "ide/Sesli-Hizli-Zeki",
|
||
"Sesli-Hizli-Zeki ", "Other-Combo"):
|
||
with self.subTest(target=target):
|
||
self.setUp()
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.h._denetimli_akis(Yanit(sse("hayir")), "[1]", None, None, "yap",
|
||
list(jev_denetim.KATMAN), self.ac, hedef=target)
|
||
self.assertEqual(self.asked, ["hayir", "bir"])
|
||
self.assertEqual(self.opened, ["Katman-1"])
|
||
|
||
def test_a_blocked_non_layer_model_goes_to_layer_1_first(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}] # the selected (non-layer) model is blocked
|
||
self.calistir(sse("Bunu yapamam."), sira=("Katman-1", "Katman-2", "Katman-3", "Katman-4"))
|
||
self.assertEqual(self.sent(), self.cevap["Katman-1"])
|
||
self.assertEqual(self.opened, ["Katman-1"])
|
||
self.assertEqual(self.asked, ["Bunu yapamam.", "bir"])
|
||
|
||
def test_blocked_layer_1_goes_to_layer_2_which_is_checked_too(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.calistir(sse("Bunu yapamam."))
|
||
self.assertEqual(self.sent(), self.cevap["Katman-2"])
|
||
self.assertNotIn(b"yapamam", self.h.wfile.getvalue())
|
||
self.assertEqual(self.opened, ["Katman-2"])
|
||
self.assertEqual(self.asked, ["Bunu yapamam.", "iki"])
|
||
self.h.send_response.assert_called_once()
|
||
|
||
def test_every_layer_blocked_ends_at_layer_4_unasked(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}] * 3
|
||
self.calistir(sse("hayir"))
|
||
self.assertEqual(self.sent(), self.cevap["Katman-4"])
|
||
self.assertEqual(self.opened, ["Katman-2", "Katman-3", "Katman-4"])
|
||
self.assertEqual(self.asked, ["hayir", "iki", "uc"]) # layer 4 is never asked
|
||
|
||
def test_blocked_layer_3_goes_straight_to_layer_4(self):
|
||
self.puanlar = [{"ret": 0.0, "sart": 0.8}]
|
||
self.calistir(sse("kendi sartimla"), sira=("Katman-4",))
|
||
self.assertEqual(self.sent(), self.cevap["Katman-4"])
|
||
self.assertEqual(self.asked, ["kendi sartimla"])
|
||
|
||
def test_a_layer_that_cannot_be_opened_is_skipped(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.cevap["Katman-2"] = None
|
||
self.calistir(sse("hayir"))
|
||
self.assertEqual(self.opened, ["Katman-2", "Katman-3"])
|
||
self.assertEqual(self.sent(), self.cevap["Katman-3"])
|
||
|
||
def test_a_layer_that_answers_with_an_error_is_skipped(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.cevap["Katman-2"] = b'event: error\ndata: {"type":"error","error":{"type":"upstream_error","message":"upstream 400"}}\n\n'
|
||
self.calistir(sse("hayir"))
|
||
self.assertEqual(self.opened, ["Katman-2", "Katman-3"])
|
||
self.assertEqual(self.sent(), self.cevap["Katman-3"])
|
||
self.assertNotIn(b"upstream 400", self.h.wfile.getvalue())
|
||
|
||
KOMUT = (b'event: response.output_item.done\ndata: {"type": "response.output_item.done", "item": '
|
||
b'{"type": "function_call", "name": "shell", "arguments": "{\\"cmd\\": \\"dir\\"}"}}\n\n')
|
||
|
||
def test_a_step_without_text_passes_without_jev(self):
|
||
body = self.KOMUT + sse()
|
||
self.calistir(body)
|
||
self.assertEqual(self.sent(), body)
|
||
self.assertEqual(self.asked, [])
|
||
self.assertEqual(self.opened, [])
|
||
|
||
def test_tool_step_cannot_be_replaced_even_if_jev_would_refuse(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
body = self.KOMUT + sse()
|
||
self.calistir(body)
|
||
self.assertEqual(self.opened, [])
|
||
self.assertEqual(self.asked, [])
|
||
self.assertEqual(self.sent(), body)
|
||
|
||
def test_tool_event_reaches_client_before_upstream_finishes(self):
|
||
first = (b'data: {"type":"response.output_item.added","item":'
|
||
b'{"type":"function_call","name":"shell","arguments":""}}\n\n')
|
||
owner = self
|
||
|
||
class Streaming(Yanit):
|
||
def __init__(self):
|
||
super().__init__(b'')
|
||
self.parts = [first, owner.KOMUT + sse()]
|
||
self.reads = 0
|
||
|
||
def read1(self, n):
|
||
self.reads += 1
|
||
if self.reads == 2:
|
||
owner.assertIn(first, owner.h.wfile.getvalue())
|
||
owner.assertEqual(owner.asked, [])
|
||
owner.assertEqual(owner.opened, [])
|
||
return super().read1(n)
|
||
|
||
self.h._denetimli_akis(Streaming(), '[1]', None, None, 'yap', ['Katman-1'], self.ac)
|
||
self.assertEqual(self.sent(), first + self.KOMUT + sse())
|
||
|
||
def test_text_with_a_tool_call_is_preparation_not_a_final_refusal(self):
|
||
body = sse('Önce dosyaları inceleyeceğim.') + self.KOMUT
|
||
self.calistir(body)
|
||
self.assertEqual(self.sent(), body)
|
||
self.assertEqual(self.asked, [])
|
||
self.assertEqual(self.opened, [])
|
||
|
||
def test_explicit_commentary_passes_without_jev(self):
|
||
body = (b'data: {"type":"response.output_item.added","item":'
|
||
b'{"type":"message","role":"assistant","phase":"commentary"}}\n\n' + sse('Araştırıyorum.'))
|
||
self.calistir(body)
|
||
self.assertEqual(self.sent(), body)
|
||
self.assertEqual(self.asked, [])
|
||
|
||
def test_an_empty_step_of_the_selected_model_goes_to_the_next_layer(self):
|
||
self.calistir(sse()) # no text, no tool call
|
||
self.assertEqual(self.asked, ["iki"]) # nothing to ask about the empty step itself
|
||
self.assertEqual(self.opened, ["Katman-2"])
|
||
self.assertEqual(self.sent(), self.cevap["Katman-2"])
|
||
|
||
def test_an_empty_layer_goes_on_to_the_one_after_it(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.cevap["Katman-2"] = sse()
|
||
self.calistir(sse("hayir"))
|
||
self.assertEqual(self.opened, ["Katman-2", "Katman-3"])
|
||
self.assertEqual(self.sent(), self.cevap["Katman-3"])
|
||
|
||
def test_an_error_of_the_selected_model_goes_to_the_next_layer(self):
|
||
self.h._denetimli_akis(None, "[1]", None, None, "yap", ["Katman-1", "Katman-2", "Katman-3", "Katman-4"], self.ac)
|
||
self.assertEqual(self.opened, ["Katman-1"])
|
||
self.assertEqual(self.sent(), self.cevap["Katman-1"])
|
||
|
||
def test_an_answer_cut_midway_goes_to_the_next_layer(self):
|
||
class Kopan(Yanit):
|
||
def read1(self, n):
|
||
if not self.parts:
|
||
raise OSError("koptu")
|
||
return self.parts.pop(0)
|
||
self.h._denetimli_akis(Kopan(sse("yarim")), "[1]", None, None, "yap", ["Katman-2", "Katman-3", "Katman-4"], self.ac)
|
||
self.assertEqual(self.opened, ["Katman-2"])
|
||
self.assertEqual(self.sent(), self.cevap["Katman-2"])
|
||
self.assertNotIn(b"yarim", self.h.wfile.getvalue())
|
||
|
||
def test_when_no_layer_opens_the_blocked_answer_is_not_delivered(self):
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
self.cevap.update({"Katman-2": None, "Katman-3": None, "Katman-4": None})
|
||
self.calistir(sse("Bunu yapamam."))
|
||
self.assertEqual(self.opened, ["Katman-2", "Katman-3", "Katman-4"])
|
||
self.assertNotIn(b"yapamam", self.h.wfile.getvalue())
|
||
self.assertEqual(self.sent(), b"") # the last layer said nothing: nothing is added
|
||
|
||
def test_the_last_layers_own_error_body_is_delivered_untouched(self):
|
||
class Hata(Exception):
|
||
def read(self):
|
||
return b'{"error": {"message": "katman 4 hatasi"}}'
|
||
self.puanlar = [{"ret": 0.9, "sart": 0.0}]
|
||
|
||
def ac(ad):
|
||
raise Hata("HTTP 400")
|
||
self.h._denetimli_akis(Yanit(sse("Bunu yapamam.")), "[1]", None, None, "yap", ["Katman-4"], ac)
|
||
self.assertEqual(self.sent(), b'{"error": {"message": "katman 4 hatasi"}}')
|
||
|
||
def test_unreachable_jev_keeps_the_answer(self):
|
||
self.puanlar = [None]
|
||
body = sse("Bunu yapamam.")
|
||
self.calistir(body)
|
||
self.assertEqual(self.sent(), body)
|
||
self.assertEqual(self.opened, [])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|