feat(settings): configure Jev evaluation and model transitions

This commit is contained in:
yilsem committed 2026-10-13 01:07:58 +03:00
1 parent 369f6c2ca8
commit 883b1dc824
20 files changed
+1195 -37

No files matched your search

+27 -3
View File
@@ -58,6 +58,7 @@ import { NotificationsSettings } from './notifications-settings'
import { PluginsSettings } from './plugins-settings'
import { PROVIDER_VIEWS, ProvidersSettings, type ProviderView } from './providers-settings'
import { SessionsSettings } from './sessions-settings'
import { StagedTransitionSettingsView } from './staged-transition-settings'
import type { SettingsPageProps, SettingsView as SettingsViewId } from './types'
const SETTINGS_VIEWS: readonly SettingsViewId[] = [
@@ -72,13 +73,14 @@ const SETTINGS_VIEWS: readonly SettingsViewId[] = [
'notifications',
'billing',
'codex-link',
'staged-transition',
'plugins',
'sessions',
'about'
]
export function SettingsView({ onClose, onConfigSaved, onMainModelChanged }: SettingsPageProps) {
const { t } = useI18n()
const { t, locale } = useI18n()
const aiturkCopy = useAiturkCopy()
const codexLinkCopy = useCodexLinkCopy()
const navigate = useNavigate()
@@ -267,7 +269,9 @@ export function SettingsView({ onClose, onConfigSaved, onMainModelChanged }: Set
id: 'aiturk-phone',
label: aiturkCopy.mobile.nav,
onSelect: () => {
void window.hermesDesktop?.aiturkMobile?.open().catch(error => notifyError(error, aiturkCopy.mobile.openFailed))
void window.hermesDesktop?.aiturkMobile
?.open()
.catch(error => notifyError(error, aiturkCopy.mobile.openFailed))
}
}
]
@@ -282,6 +286,13 @@ export function SettingsView({ onClose, onConfigSaved, onMainModelChanged }: Set
id: 'codex-link',
label: codexLinkCopy.nav,
onSelect: () => setActiveView('codex-link')
},
{
active: activeView === 'staged-transition',
icon: Link,
id: 'staged-transition',
label: locale === 'tr' ? 'Kademeli Geçiş' : 'Staged Transition',
onSelect: () => setActiveView('staged-transition')
}
]
: []),
@@ -338,7 +349,18 @@ export function SettingsView({ onClose, onConfigSaved, onMainModelChanged }: Set
onSelect: () => setActiveView('about')
}
],
[activeView, keysView, providerView, t, aiturkCopy, codexLinkCopy, setActiveView, openProviderView, openKeysView]
[
activeView,
keysView,
providerView,
t,
locale,
aiturkCopy,
codexLinkCopy,
setActiveView,
openProviderView,
openKeysView
]
)
// Type-to-search: printable keystrokes on the Settings surface (outside any
@@ -459,6 +481,8 @@ export function SettingsView({ onClose, onConfigSaved, onMainModelChanged }: Set
<PluginsSettings />
) : activeView === 'codex-link' ? (
<CodexLinkSettings />
) : activeView === 'staged-transition' ? (
<StagedTransitionSettingsView />
) : (
<SessionsSettings />
)
@@ -0,0 +1,71 @@
import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { I18nProvider } from '@/i18n/context'
import defaults from '../../../assets/codex-bridge/jev-settings-defaults.json'
import { StagedTransitionSettingsView } from './staged-transition-settings'
const get = vi.fn()
const set = vi.fn()
const state = () => ({
settings: structuredClone(defaults),
keyConfigured: true,
modelsAvailable: true,
models: [
{ id: 'A', name: 'Model A' },
{ id: 'B', name: 'Model B' }
]
})
beforeEach(() => {
;(window as unknown as { hermesDesktop: unknown }).hermesDesktop = {
codexLink: { getStagedTransition: get, setStagedTransition: set }
}
get.mockResolvedValue(state())
set.mockImplementation(async patch => ({ ...state(), settings: patch.settings }))
})
afterEach(() => {
cleanup()
vi.resetAllMocks()
})
const show = () =>
render(
<I18nProvider configClient={null} initialLocale="tr">
<StagedTransitionSettingsView />
</I18nProvider>
)
describe('staged transition settings UI', () => {
it('adds and removes model routing and always-pass rows', async () => {
show()
await screen.findByText('Kademeli Geçiş')
fireEvent.click(screen.getByText('+ Geçiş kuralı ekle'))
expect(screen.getByLabelText('İtiraz edilen model 4')).toBeTruthy()
expect(screen.getByLabelText('Kullanılacak model 4')).toBeTruthy()
fireEvent.click(screen.getByLabelText('Geçiş kuralı 4 sil'))
expect(screen.queryByLabelText('İtiraz edilen model 4')).toBeNull()
fireEvent.click(screen.getByText('+ Her zaman geç kuralı ekle'))
expect(screen.getByLabelText('Her zaman geçen model 3')).toBeTruthy()
fireEvent.click(screen.getByLabelText('Her zaman geç kuralı 3 sil'))
fireEvent.change(screen.getByLabelText('İsteği yerine getirmeme denetim metni'), {
target: { value: 'custom prompt' }
})
fireEvent.click(screen.getByText('Kaydet'))
await screen.findByText('Kaydedildi. Yeni değerlendirmelerde uygulanır.')
expect(set.mock.calls[0][0].settings.questions.ret).toBe('custom prompt')
expect(set.mock.calls[0][0]).not.toHaveProperty('apiKey')
})
it('retains user edits if persistence fails', async () => {
set.mockRejectedValue(new Error('disk error'))
show()
await screen.findByText('Kademeli Geçiş')
fireEvent.change(screen.getByLabelText('Jev modeli'), { target: { value: 'new-model' } })
fireEvent.click(screen.getByText('Kaydet'))
await waitFor(() => expect(screen.getByRole('alert').textContent).toContain('Kaydedilemedi'))
expect((screen.getByLabelText('Jev modeli') as HTMLInputElement).value).toBe('new-model')
})
})
@@ -0,0 +1,463 @@
import { useEffect, useState } from 'react'
import { Button } from '@/components/ui/button'
import { Input } from '@/components/ui/input'
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'
import { Switch } from '@/components/ui/switch'
import { Textarea } from '@/components/ui/textarea'
import { useI18n } from '@/i18n/context'
import { Link } from '@/lib/icons'
import defaultSettings from '../../../assets/codex-bridge/jev-settings-defaults.json'
import type { StagedTransitionSettings, StagedTransitionState } from '../../../electron/staged-transition-types'
import { ListRow, SettingsContent, SettingsSection } from './primitives'
export function StagedTransitionSettingsView() {
const { locale } = useI18n()
const tr = locale === 'tr'
const text = (turkish: string, english: string) => (tr ? turkish : english)
const api = window.hermesDesktop?.codexLink
const [state, setState] = useState<StagedTransitionState | null>(null)
const [draft, setDraft] = useState<StagedTransitionSettings | null>(null)
const [key, setKey] = useState('')
const [busy, setBusy] = useState(false)
const [message, setMessage] = useState('')
const [error, setError] = useState(false)
const adopt = (next: StagedTransitionState) => {
setState(next)
setDraft(next.settings)
}
useEffect(() => {
let active = true
if (api?.getStagedTransition) {
api.getStagedTransition().then(
next => {
if (active) {
adopt(next)
}
},
() => {
if (active) {
setError(true)
setMessage(tr ? 'Ayarlar okunamadı.' : 'Could not read settings.')
}
}
)
}
return () => {
active = false
}
}, [api, tr])
if (!api?.getStagedTransition) {
return (
<SettingsContent>
<p>
{text('Bu bölüm Windows masaüstü uygulamasında kullanılabilir.', 'Available in the Windows desktop app.')}
</p>
</SettingsContent>
)
}
if (!draft) {
return (
<SettingsContent>
<p role={error ? 'alert' : 'status'}>{message || text('Yükleniyor…', 'Loading…')}</p>
</SettingsContent>
)
}
const change = <K extends keyof StagedTransitionSettings>(name: K, value: StagedTransitionSettings[K]) =>
setDraft({ ...draft, [name]: value })
const models = new Map((state?.models ?? []).map(model => [model.id, model.name]))
const storedModels = [
draft.fallbackModel,
...draft.exemptModels,
...draft.rejectionRules.flatMap(rule => [rule.from, rule.to])
]
for (const id of storedModels) {
if (id && !models.has(id)) {
models.set(id, `${id} (${text('kayıtlı', 'saved')})`)
}
}
const modelSelect = (label: string, value: string, onChange: (value: string) => void) => (
<div className="w-full min-w-0 sm:w-auto sm:min-w-48 sm:flex-1">
<Select onValueChange={onChange} value={value}>
<SelectTrigger aria-label={label}>
<SelectValue placeholder={text('Model seçin', 'Select a model')} />
</SelectTrigger>
<SelectContent>
{Array.from(models, ([id, name]) => (
<SelectItem key={id} value={id}>
{name}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
)
const number = (
name: 'timeoutSeconds' | 'maxConcurrent' | 'queueWaitSeconds',
label: string,
min: number,
max: number
) => (
<ListRow
action={
<Input
aria-label={label}
max={max}
min={min}
onChange={event => change(name, Number(event.target.value))}
type="number"
value={draft[name]}
/>
}
title={label}
/>
)
const save = async () => {
setBusy(true)
setMessage('')
setError(false)
try {
adopt(await api.setStagedTransition({ settings: draft, ...(key.trim() ? { apiKey: key.trim() } : {}) }))
setKey('')
setMessage(text('Kaydedildi. Yeni değerlendirmelerde uygulanır.', 'Saved. Applies to new evaluations.'))
} catch (failure) {
setError(true)
const cycle = String(failure).includes('cycle')
setMessage(
cycle
? text(
'Geçiş kuralları döngü oluşturuyor. Aynı modele geri dönen geçişi değiştirin.',
'Transition rules form a cycle. Change the rule that returns to an earlier model.'
)
: text(
'Kaydedilemedi. Alanları ve sayı aralıklarını kontrol edin; önceki ayarlar korunuyor.',
'Could not save. Check the fields and number ranges; previous settings are preserved.'
)
)
} finally {
setBusy(false)
}
}
return (
<SettingsContent>
<fieldset className="min-w-0" disabled={busy}>
<SettingsSection icon={Link} title={text('Kademeli Geçiş', 'Staged Transition')}>
<ListRow
action={
<Switch
aria-label={text('Jev denetimi', 'Jev evaluation')}
checked={draft.enabled}
onCheckedChange={value => change('enabled', value)}
/>
}
description={text(
'Köprüdeki ek modellerin nihai cevaplarını denetler. Uygun bulunmayan cevap sıradaki katmanda yeniden hazırlanır.',
'Evaluates final answers from bridge models. Rejected answers are retried with the next layer.'
)}
title={text('Jev denetimi ve kademeli geçiş', 'Jev evaluation and staged transition')}
/>
<ListRow
action={
<Input
aria-label={text('Jev modeli', 'Jev model')}
onChange={e => change('model', e.target.value)}
value={draft.model}
/>
}
title={text('Jev modeli', 'Jev model')}
/>
<ListRow
action={
<Input
aria-label={text('API adresi', 'API endpoint')}
onChange={e => change('endpoint', e.target.value)}
type="url"
value={draft.endpoint}
/>
}
title={text('API adresi', 'API endpoint')}
/>
<ListRow
action={
<Input
aria-label={text('TypeSafe API anahtarı', 'TypeSafe API key')}
autoComplete="new-password"
onChange={e => setKey(e.target.value)}
type="password"
value={key}
/>
}
description={
state?.keyConfigured
? text(
'Anahtar tanımlı. Değiştirmek için yeni anahtarı girin.',
'Key configured. Enter a new key to replace it.'
)
: text('Henüz anahtar tanımlı değil.', 'No key configured.')
}
title={text('TypeSafe API anahtarı', 'TypeSafe API key')}
/>
</SettingsSection>
<SettingsSection icon={Link} title={text('Denetim kuralları', 'Evaluation rules')}>
{(['ret', 'sart'] as const).map(id => {
const label =
id === 'ret'
? text('İsteği yerine getirmeme', 'Request not fulfilled')
: text('İsteği kendiliğinden değiştirme', 'Unrequested changes')
return (
<div key={id}>
<ListRow
action={
<Input
aria-label={`${label} ${text('eşiği', 'threshold')}`}
max={1}
min={0}
onChange={e => change('thresholds', { ...draft.thresholds, [id]: Number(e.target.value) })}
step={0.01}
type="number"
value={draft.thresholds[id]}
/>
}
description={text(
'0–1 arasında geçiş eşiği. Olasılık bu değere ulaşırsa sıradaki katman denenir.',
'Transition threshold from 0 to 1. A probability at or above this value tries the next layer.'
)}
title={label}
/>
<Textarea
aria-label={`${label} ${text('denetim metni', 'instructions')}`}
onChange={e => change('questions', { ...draft.questions, [id]: e.target.value })}
rows={4}
value={draft.questions[id]}
/>
{(['true', 'false'] as const).map(outcome => (
<label className="block py-2" key={outcome}>
<span>
{outcome === 'true'
? text(
'“Evet, sorun var” ne demek? (isteğe bağlı)',
'What does “yes, there is a problem” mean? (optional)'
)
: text(
'“Hayır, sorun yok” ne demek? (isteğe bağlı)',
'What does “no problem” mean? (optional)'
)}
</span>
<Textarea
aria-label={`${label} ${outcome}`}
onChange={e =>
change('criteria', {
...draft.criteria,
[id]: { ...draft.criteria[id], [outcome]: e.target.value }
})
}
rows={2}
value={draft.criteria[id][outcome]}
/>
</label>
))}
</div>
)
})}
</SettingsSection>
<SettingsSection
icon={Link}
title={text('Olasılık, onay ve sıcaklık', 'Probability, approval and temperature')}
>
<p>
{text(
'Jev her denetim sorusu için 0–1 arasında sorun olasılığı verir. Her iki olasılık da kendi itiraz eşiğinin altındaysa cevap geçer. Örneğin 0,70 eşiği, sorun olasılığı %70 veya üstündeyse geçiş demektir; %70 kalite garantisi değildir.',
'Jev returns a problem probability from 0 to 1 for each question. An answer passes when both probabilities are below their rejection thresholds. A threshold of 0.70 triggers transition at 70% problem probability; it is not a quality guarantee.'
)}
</p>
<ListRow
description={text('TypeSafe API bu parametreyi sunmuyor.', 'Not exposed by the TypeSafe API.')}
title={text('Sıcaklık', 'Temperature')}
/>
<ListRow
description={text(
'Kullanılan Noul (evet/hayır) türünde ayrı güven skoru yoktur. Karar yukarıdaki olasılık eşikleriyle verilir.',
'The Noul (yes/no) type has no separate confidence score. Decisions use the probability thresholds above.'
)}
title={text('Ayrı emin olma / güven değeri', 'Separate confidence value')}
/>
</SettingsSection>
<SettingsSection icon={Link} title={text('Geçiş kuralları', 'Transition rules')}>
{!state?.modelsAvailable && (
<p role="status">
{text(
'Canlı model listesi alınamadı. Kayıtlı seçimler korunuyor; Codex Bağlama açıkken listeyi yenileyin.',
'Live model list unavailable. Saved selections are preserved; refresh with Codex Link enabled.'
)}
</p>
)}
<Button
onClick={async () => {
try {
const next = await api.getStagedTransition()
setState(next)
} catch {
setError(true)
setMessage(text('Model listesi alınamadı.', 'Could not load models.'))
}
}}
variant="outline"
>
{text('Model listesini yenile', 'Refresh models')}
</Button>
<ListRow
action={modelSelect(text('Genel geçiş modeli', 'Default fallback model'), draft.fallbackModel, value =>
change('fallbackModel', value)
)}
description={text(
'Modele özel kural yoksa seçilen model kullanılır.',
'Uses this model when there is no model-specific rule.'
)}
title={text('Jev herhangi bir modele itiraz ederse', 'When Jev rejects any model')}
/>
{draft.rejectionRules.map((rule, index) => (
<div className="flex flex-wrap items-center gap-2 py-2" key={index}>
{modelSelect(text(`İtiraz edilen model ${index + 1}`, `Rejected model ${index + 1}`), rule.from, value =>
change(
'rejectionRules',
draft.rejectionRules.map((r, i) => (i === index ? { ...r, from: value } : r))
)
)}
<span>{text('modeline Jev itiraz ederse', 'rejected by Jev →')}</span>
{modelSelect(text(`Kullanılacak model ${index + 1}`, `Fallback model ${index + 1}`), rule.to, value =>
change(
'rejectionRules',
draft.rejectionRules.map((r, i) => (i === index ? { ...r, to: value } : r))
)
)}
<span>{text('modeli kullanılsın', 'use this model')}</span>
<Button
aria-label={text(`Geçiş kuralı ${index + 1} sil`, `Remove rule ${index + 1}`)}
onClick={() =>
change(
'rejectionRules',
draft.rejectionRules.filter((_, i) => i !== index)
)
}
variant="ghost"
>
{text('Sil', 'Remove')}
</Button>
</div>
))}
<Button
onClick={() => change('rejectionRules', [...draft.rejectionRules, { from: '', to: '' }])}
variant="outline"
>
{text('+ Geçiş kuralı ekle', '+ Add transition rule')}
</Button>
</SettingsSection>
<SettingsSection icon={Link} title={text('Jev’den her zaman geçsin', 'Always pass Jev')}>
<p>
{text(
'Bu modeller Jev denetimine gönderilmez. Bu seçim, modele özel ve genel itiraz kurallarından önce uygulanır.',
'These models bypass Jev evaluation. This takes priority over model-specific and default rejection rules.'
)}
</p>
{draft.exemptModels.map((model, index) => (
<div className="flex flex-wrap items-center gap-2 py-2" key={index}>
{modelSelect(text(`Her zaman geçen model ${index + 1}`, `Always-pass model ${index + 1}`), model, value =>
change(
'exemptModels',
draft.exemptModels.map((m, i) => (i === index ? value : m))
)
)}
<span>{text('modeli Jev’den her zaman geçsin', 'always passes Jev')}</span>
<Button
aria-label={text(`Her zaman geç kuralı ${index + 1} sil`, `Remove always-pass rule ${index + 1}`)}
onClick={() =>
change(
'exemptModels',
draft.exemptModels.filter((_, i) => i !== index)
)
}
variant="ghost"
>
{text('Sil', 'Remove')}
</Button>
</div>
))}
<Button onClick={() => change('exemptModels', [...draft.exemptModels, ''])} variant="outline">
{text('+ Her zaman geç kuralı ekle', '+ Add always-pass rule')}
</Button>
</SettingsSection>
<SettingsSection icon={Link} title={text('Paralellik ve bekleme', 'Concurrency and waiting')}>
{number(
'maxConcurrent',
text('Eşzamanlı Jev isteği (0: yerel sınır yok)', 'Concurrent Jev requests (0: no local limit)'),
0,
512
)}
{number(
'queueWaitSeconds',
text('Sırada en fazla bekleme (saniye)', 'Maximum queue wait (seconds)'),
1,
3600
)}
{number('timeoutSeconds', text('Jev yanıt zaman aşımı (saniye)', 'Jev response timeout (seconds)'), 1, 3600)}
<ListRow
action={
<Select
onValueChange={value => change('failureAction', value as 'pass' | 'next')}
value={draft.failureAction}
>
<SelectTrigger aria-label={text('Jev’e ulaşılamadığında', 'When Jev is unavailable')}>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="pass">{text('Mevcut cevabı ilet', 'Pass the current answer')}</SelectItem>
<SelectItem value="next">{text('Sıradaki katmanı dene', 'Try the next layer')}</SelectItem>
</SelectContent>
</Select>
}
title={text('Jev’e ulaşılamadığında', 'When Jev is unavailable')}
/>
</SettingsSection>
<Button onClick={() => void save()}>{busy ? text('Kaydediliyor…', 'Saving…') : text('Kaydet', 'Save')}</Button>
<Button
onClick={() => {
setDraft(structuredClone(defaultSettings) as StagedTransitionSettings)
setMessage(
text(
'Varsayılanlar forma yüklendi. Uygulamak için Kaydet’e basın.',
'Defaults loaded into the form. Save to apply.'
)
)
setError(false)
}}
variant="ghost"
>
{text('Varsayılanları yükle', 'Load defaults')}
</Button>
{message && (
<p className="py-3" role={error ? 'alert' : 'status'}>
{message}
</p>
)}
</fieldset>
</SettingsContent>
)
}
+1
View File
@@ -16,6 +16,7 @@ export type SettingsView =
| 'plugins'
| 'providers'
| 'sessions'
| 'staged-transition'
| `config:${string}`
export type EnvPatch = Partial<Pick<EnvVarInfo, 'is_set' | 'redacted_value'>>