From 86a662193645f9dcc4a991cb7a422ed2ffa770c4 Mon Sep 17 00:00:00 2001 From: yilsem Date: Thu, 15 Oct 2026 08:38:52 +0300 Subject: [PATCH] feat(chrome): let the IDE agent use the user's Chrome and beta.42 The AITURK Chrome extension now registers the user's Chrome as the desktop backend's browser controller, the way the ChatGPT extension lets its agent work in the user's browser. The agent's existing browser tools (navigate, snapshot, click, type, scroll, back, press, get_images, vision) run in its own background tabs grouped as "AITURK", or a tab the user hands over. Backend: a local desktop lane in the browser-control broker. One controller registered from the loopback desktop transport is offered to every desktop session as a lane template; each session still gets its own exact scope on first use. The lane has its own switch (browser.extension_control.local_desktop, on by default); with no extension registered, or after it disconnects, the normal browser backend runs unchanged. browser_vision turns the extension's capture into the usual native/auxiliary vision result. Bridge: forwards controller commands only to a connection that registered, refuses unknown or oversized commands, relays results. Extension: service-worker controller with its own native connection, trusted chrome.debugger input (press/release only: a hover move waits 5 s on background tabs), refusal of password and payment-card fields and of browser, extension, file and Web Store pages, panel switch, Stop, tab hand-over, recent actions, Alt+Shift+A and right-click "send to AITURK". Cancelling Chrome's debugging bar stops the agent. Existing panel flows and privacy rules are unchanged. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/ci.yml | 4 + apps/chrome/KURULUM.txt | 28 +- apps/chrome/README.md | 17 +- apps/chrome/extension/background.js | 41 ++ apps/chrome/extension/controller.js | 646 ++++++++++++++++++ apps/chrome/extension/icon128.png | Bin 0 -> 4964 bytes apps/chrome/extension/icon16.png | Bin 0 -> 589 bytes apps/chrome/extension/icon32.png | Bin 0 -> 1172 bytes apps/chrome/extension/icon48.png | Bin 0 -> 1848 bytes apps/chrome/extension/manifest.json | 13 +- apps/chrome/extension/page-agent.js | 341 +++++++++ apps/chrome/extension/panel.css | 2 +- apps/chrome/extension/panel.html | 8 +- apps/chrome/extension/panel.js | 92 ++- apps/chrome/tests/agent.test.mjs | 339 +++++++++ apps/chrome/tests/bridge.test.mjs | 49 ++ apps/chrome/tests/panel.test.mjs | 61 +- apps/desktop/electron/chrome-bridge.mjs | 69 +- apps/desktop/package.json | 2 +- gateway/browser_control_broker.py | 256 ++++++- hermes_cli/config_defaults.py | 4 + hermes_cli/web_server.py | 4 + package-lock.json | 2 +- tests/gateway/test_browser_control_api.py | 2 + .../test_browser_control_local_lane.py | 345 ++++++++++ .../test_browser_extension_router_wiring.py | 3 +- tools/browser_extension_router.py | 54 +- tools/browser_tool.py | 397 +++++++---- tui_gateway/methods_browser_control.py | 134 ++++ tui_gateway/server.py | 11 + tui_gateway/ws.py | 7 + .../docs/user-guide/features/api-server.md | 19 +- 32 files changed, 2755 insertions(+), 195 deletions(-) create mode 100644 apps/chrome/extension/controller.js create mode 100644 apps/chrome/extension/icon128.png create mode 100644 apps/chrome/extension/icon16.png create mode 100644 apps/chrome/extension/icon32.png create mode 100644 apps/chrome/extension/icon48.png create mode 100644 apps/chrome/extension/page-agent.js create mode 100644 apps/chrome/tests/agent.test.mjs create mode 100644 tests/gateway/test_browser_control_local_lane.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index c1a3285..2bdec20 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -39,6 +39,8 @@ jobs: run: '"$RUNNER_TEMP/ci-venv/bin/python" -m pytest tests/test_install_ps1_reuse_healthy_venv.py tests/test_install_ps1_python_fallback_venv.py -q -p no:cacheprovider -rf' - name: Project scoped and bounded search regressions run: '"$RUNNER_TEMP/ci-venv/bin/python" -m pytest tests/tools/test_search_project_scope.py tests/tools/test_search_files_cpu_windows.py tests/tools/test_search_zero_match_and_multipath.py tests/tools/test_search_giant_line_containment.py tests/tools/test_search_budget_truncation.py -q -p no:cacheprovider -rf' + - name: Desktop browser control lane (AITURK Chrome) + run: '"$RUNNER_TEMP/ci-venv/bin/python" -m pytest tests/gateway/test_browser_control_local_lane.py tests/gateway/test_browser_control_broker.py tests/gateway/test_browser_control_broker_hardening.py tests/gateway/test_browser_control_cloud.py tests/tools/test_browser_extension_router.py tests/tools/test_browser_extension_router_wiring.py -q -p no:cacheprovider -rf' - name: Native watch and speech mux pytest regressions run: '"$RUNNER_TEMP/ci-venv/bin/python" -m pytest apps/desktop/assets/codex-bridge/test_bekci_watch.py apps/desktop/assets/codex-bridge/test_native_speech_mux.py apps/desktop/assets/codex-bridge/test_live_limits.py -q -p no:cacheprovider -rf' - name: Mobile and desktop startup regression tests @@ -63,6 +65,8 @@ jobs: github-server-url: https://gitea.turkservis.online - name: Install locked source dependencies run: npm ci --no-audit --no-fund + - name: Chrome extension, agent control and bridge tests + run: node --test apps/chrome/tests/agent.test.mjs apps/chrome/tests/bridge.test.mjs apps/chrome/tests/panel.test.mjs - name: Install Windows launcher cross compiler and framework reference run: | set -eu diff --git a/apps/chrome/KURULUM.txt b/apps/chrome/KURULUM.txt index 7f40420..1b66d81 100644 --- a/apps/chrome/KURULUM.txt +++ b/apps/chrome/KURULUM.txt @@ -1,7 +1,8 @@ -AITURK Chrome — Windows beta 0.1.0 +AITURK Chrome — Windows beta 0.2.0 GEREKSİNİMLER -Windows 10/11, Chrome 116 veya üzeri, AITURK IDE 1.0.0-beta.7 veya üzeri. +Windows 10/11, Chrome 116 veya üzeri, AITURK IDE 1.0.0-beta.42 veya üzeri +(ajanın Chrome'u kullanması için; konuşma paneli eski IDE'lerde de çalışır). Eski IDE sürümleri bu eklentiye bağlanamaz. Yerel IDE açık olmalıdır. İlk beta yalnızca IDE'nin etkin yerel profilini destekler; uzak sunucu/bulut konuşmaları ve başka bilgisayardaki konuşmalar bu sürümde desteklenmez. @@ -27,6 +28,26 @@ Görüntülenen uzun mesajlar kısaltılabilir. Modelin mevcut bağlamı değiş "Yenile" konuşma listesini ve seçili geçmişi tekrar okur. Ctrl+Enter mesaj gönderir. Enter yeni satır ekler. +AJAN BU CHROME'U KULLANSIN (YENİ) +Paneldeki "Ajan bu Chrome'u kullanabilsin" anahtarı açıkken (varsayılan açık) +AITURK IDE ajanı tarayıcı işlerini bu Chrome'da yapar: sayfa açar, okur, +tıklar, yazar, kaydırır, geri gider, tuşa basar, görselleri listeler ve ekran +görüntüsüne bakar. Panel kapalı olsa da çalışır; IDE açık olmalıdır. +- Ajan yalnız kendi sekmelerinde çalışır; bunlar "AITURK" sekme grubunda, + arka planda açılır. Kendi sekmenizi vermek için panelde "Bu sekmeyi ajana + ver"e ya da sağ tıkta "Bu sekmeyi AITURK ajanına ver"e basın. +- Ajan çalışırken Chrome üstte "hata ayıklama" uyarısı gösterir. Uyarıdaki + İptal, paneldeki Durdur ya da anahtarı kapatmak ajanı hemen durdurur. +- Ajan şifre ve kart numarası alanlarına yazmaz; bu adımları size bırakır ve + bildirim gösterir. chrome://, eklenti, dosya ve Web Mağazası sayfalarına girmez. +- Anahtar kapalıyken ya da Chrome kapalıyken ajan kendi tarayıcısını kullanır. +- Panelde "Ajanın son işlemleri" listesi ajanın ne yaptığını gösterir. + +KISAYOL VE SAĞ TIK +Alt+Shift+A paneli açar (chrome://extensions/shortcuts üzerinden değişir). +Sağ tık: "AITURK'e gönder: seçili metin" / "bu sayfa" paneli açar ve metni +mesajınıza ek olarak hazırlar; konuşma seçince gönderebilirsiniz. + SAYFA İÇERİĞİ VE GİZLİLİK Eklenti gezinme geçmişini, çerezleri, form değerlerini veya parolaları toplamaz. Sayfa/seçim yalnızca "Sayfayı / seçimi ekle" tıklandığında okunur; siz Gönder'e @@ -41,7 +62,8 @@ SORUN GİDERME "Native messaging host not found": bağlantı kurulumunu yeniden çalıştırın. "IDE bağlantısı bulunamadı": Chrome destekli IDE sürümü açık ve yerel bağlantıda olmalıdır. IDE'yi yeniden başlatıp eklentide Yeniden bağlan'a basın. -"Cannot access contents": ilgili sekmede AITURK araç çubuğu simgesine tıklayın. +"Cannot access contents": eklentinin site erişimi kapalı olabilir; chrome://extensions +üzerinden AITURK için "Tüm sitelerde" erişimi açın. chrome://, Chrome Web Store, PDF görüntüleyici gibi korumalı sayfalar okunamaz. Zaman aşımında mesaj otomatik tekrar gönderilmez. Çifte gönderimi önlemek için önce IDE geçmişini kontrol edin. Taslağınız panel kapanana kadar korunur. diff --git a/apps/chrome/README.md b/apps/chrome/README.md index 7ff52e1..d4db63c 100644 --- a/apps/chrome/README.md +++ b/apps/chrome/README.md @@ -1,7 +1,7 @@ # AITURK Chrome beta Windows / Chrome MV3 side panel for existing local AITURK IDE conversations. -Requires AITURK IDE 1.0.0-beta.9 or later; see `KURULUM.txt` for Turkish setup. +Requires AITURK IDE 1.0.0-beta.9 or later (agent browser control: 1.0.0-beta.42); see `KURULUM.txt` for Turkish setup. The extension talks only to `online.turkservis.aiturk` using Chrome native messaging. The per-user .NET Framework host authenticates to Electron's random @@ -19,6 +19,20 @@ context is not truncated by this display policy. Approvals and clarifications remain in the IDE. A backend event is stamped once and delivered to every attached viewer so the desktop can still show and answer these requests. +## Agent browser control + +The service worker keeps its own native connection and registers this Chrome +with the IDE backend as the local desktop browser controller +(`browser.controller.local.*`, see the API server docs). The IDE agent's +existing `browser_*` tools (navigate, snapshot, click, type, scroll, back, +press, get_images, vision) then run here, in agent tabs grouped as "AITURK" +or a tab the user hands over; when the switch is off or Chrome is closed the +agent keeps its normal browser. Clicks and key presses use trusted +`chrome.debugger` input; password and payment-card fields are refused; +chrome://, extension, file and Web Store pages are refused. Cancelling +Chrome's debugging bar, the panel's Stop button or the switch turns control +off and cancels pending commands. + ## Build and verification From the repository root, on Windows: @@ -39,6 +53,7 @@ UI tests require the existing desktop workspace's jsdom dependency: ```powershell npm ci --workspace apps/desktop --include-workspace-root node --test apps/chrome/tests/panel.test.mjs +node --test apps/chrome/tests/agent.test.mjs ``` Exercise the production FastAPI session router and real temporary SQLite stores diff --git a/apps/chrome/extension/background.js b/apps/chrome/extension/background.js index 5d9d34e..b5bfad3 100644 --- a/apps/chrome/extension/background.js +++ b/apps/chrome/extension/background.js @@ -1 +1,42 @@ +import { createController } from './controller.js' + +const controller = createController({ chrome }) + chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {}) + +// The side panel talks to the controller; nothing else may. +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + const type = String(message?.type || '') + if (sender.id !== chrome.runtime.id || !type.startsWith('control.') || type === 'control.state') return false + controller.onMessage(message, sender).then(sendResponse, error => sendResponse({ failure: String(error?.message || error) })) + return true +}) +chrome.debugger.onDetach.addListener((source, reason) => controller.onDebuggerDetach(source, reason)) +chrome.tabs.onRemoved.addListener(tabId => { void controller.onTabRemoved(tabId) }) +chrome.alarms.onAlarm.addListener(alarm => controller.onAlarm(alarm.name)) + +const MENU = [ + { id: 'aiturk-selection', title: 'AITURK’e gönder: seçili metin', contexts: ['selection'] }, + { id: 'aiturk-page', title: 'AITURK’e gönder: bu sayfa', contexts: ['page'] }, + { id: 'aiturk-share', title: 'Bu sekmeyi AITURK ajanına ver', contexts: ['page'] }, +] +chrome.runtime.onInstalled.addListener(() => { + chrome.contextMenus.removeAll(() => { for (const item of MENU) chrome.contextMenus.create(item) }) +}) +chrome.contextMenus.onClicked.addListener((info, tab) => { + if (!tab?.windowId) return + // Opening the panel must happen inside the click's user gesture, before any await. + chrome.sidePanel.open({ windowId: tab.windowId }).catch(() => {}) + if (info.menuItemId === 'aiturk-share') { + void chrome.storage.local.get('selectedSession') + .then(saved => controller.shareTab(tab, typeof saved.selectedSession === 'string' ? saved.selectedSession : null)) + .catch(error => chrome.storage.session.set({ pendingNotice: String(error?.message || error) })) + return + } + void chrome.storage.session.set({ pendingContext: { + kind: info.menuItemId === 'aiturk-selection' ? 'selection' : 'page', tabId: tab.id, + text: info.selectionText ? info.selectionText.slice(0, 24000) : '', at: Date.now(), + } }) +}) + +void controller.connect() diff --git a/apps/chrome/extension/controller.js b/apps/chrome/extension/controller.js new file mode 100644 index 0000000..23d1ae6 --- /dev/null +++ b/apps/chrome/extension/controller.js @@ -0,0 +1,646 @@ +// Lets the AITURK IDE agent use this Chrome, the way the ChatGPT extension +// lets its agent work in the user's browser. Runs in the service worker and +// keeps its own native connection, so it works with the side panel closed. +// +// Boundaries: the agent works only in its own tabs (grouped as "AITURK") or a +// tab the user explicitly hands over; never on chrome://, extension, file or +// Web Store pages; never types into password or payment-card fields. Chrome +// shows its own "debugging" bar while the agent is in control; cancelling it, +// the panel's Stop button or the switch turns control off at once. + +import { + annotateRefs, clickWithEvents, listImages, locateRef, prepareTyping, readValue, scrollPage, snapshotPage, typeWithEvents, +} from './page-agent.js' + +export const HOST = 'online.turkservis.aiturk' +export const CAPABILITIES = Object.freeze(['browser_navigate', 'browser_snapshot', 'browser_click', 'browser_type', + 'browser_scroll', 'browser_back', 'browser_press', 'browser_get_images', 'browser_vision']) +const COMMAND_BUDGET_MS = 27000 +const LOAD_WAIT_MS = 20000 +const DEBUGGER_IDLE_MS = 45000 +const SCREENSHOT_LIMIT = 380000 +const LOG_SIZE = 30 + +export class ToolError extends Error {} + +const KEY_ALIASES = { esc: 'Escape', return: 'Enter', up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight', + space: ' ', spacebar: ' ', del: 'Delete', pgup: 'PageUp', pgdn: 'PageDown', ctrl: 'Control', cmd: 'Meta', command: 'Meta', option: 'Alt' } +const NAMED_KEYS = { Enter: [13, 'Enter', '\r'], Tab: [9, 'Tab'], Escape: [27, 'Escape'], Backspace: [8, 'Backspace'], + Delete: [46, 'Delete'], ArrowUp: [38, 'ArrowUp'], ArrowDown: [40, 'ArrowDown'], ArrowLeft: [37, 'ArrowLeft'], + ArrowRight: [39, 'ArrowRight'], Home: [36, 'Home'], End: [35, 'End'], PageUp: [33, 'PageUp'], PageDown: [34, 'PageDown'], + ' ': [32, 'Space', ' '], F5: [116, 'F5'] } +const MODIFIERS = { Alt: 1, Control: 2, Meta: 4, Shift: 8 } + +// "Control+a" → one CDP key description; unknown names fail before any input. +export function parseKey(value) { + const parts = String(value || '').split('+').map(part => part.trim()).filter(Boolean) + if (!parts.length || parts.length > 4) throw new ToolError('Tuş adı geçersiz. Örnek: Enter, Tab, Escape, Control+a') + let modifiers = 0 + const keyName = parts.pop() + for (const raw of parts) { + const name = KEY_ALIASES[raw.toLowerCase()] || raw[0].toUpperCase() + raw.slice(1).toLowerCase() + if (!MODIFIERS[name]) throw new ToolError(`Bilinmeyen değiştirici tuş: ${raw}`) + modifiers |= MODIFIERS[name] + } + const alias = KEY_ALIASES[keyName.toLowerCase()] + const name = alias || (keyName.length === 1 ? keyName : keyName[0].toUpperCase() + keyName.slice(1)) + const named = NAMED_KEYS[name] || NAMED_KEYS[Object.keys(NAMED_KEYS).find(key => key.toLowerCase() === name.toLowerCase())] + if (named) { + const [code, keyCode, text] = named + return { key: name === ' ' ? ' ' : keyCode, code: keyCode === ' ' ? 'Space' : keyCode, windowsVirtualKeyCode: code, + text: modifiers & ~MODIFIERS.Shift ? undefined : text, modifiers } + } + if (name.length !== 1) throw new ToolError(`Bilinmeyen tuş: ${keyName}`) + const upper = name.toUpperCase() + const letter = /[A-Z]/.test(upper), digit = /[0-9]/.test(name) + return { key: name, code: letter ? `Key${upper}` : digit ? `Digit${name}` : '', + windowsVirtualKeyCode: letter || digit ? upper.charCodeAt(0) : name.charCodeAt(0), + text: modifiers & ~MODIFIERS.Shift ? undefined : name, modifiers } +} + +// Only ordinary web pages. Local development servers are allowed; browser, +// extension, file and Web Store pages are not. +export function checkUrl(value, { allowBlank = false } = {}) { + let text = String(value || '').trim() + if (!text) throw new ToolError('Adres boş.') + if (!/^[a-z][a-z0-9+.-]*:/i.test(text)) text = `https://${text}` + let url + try { url = new URL(text) } catch { throw new ToolError(`Geçersiz adres: ${String(value).slice(0, 200)}`) } + if (allowBlank && url.href === 'about:blank') return url.href + if (!['http:', 'https:'].includes(url.protocol)) { + throw new ToolError(`Ajan bu tür sayfada çalışamaz (${url.protocol}). Yalnız http/https sayfaları açılabilir.`) + } + if (/^(chrome\.google\.com|chromewebstore\.google\.com)$/i.test(url.hostname) && + (url.hostname.startsWith('chromewebstore') || url.pathname.startsWith('/webstore'))) { + throw new ToolError('Chrome Web Mağazası sayfalarında ajan çalışamaz.') + } + if (url.username || url.password) throw new ToolError('Adreste kullanıcı adı/şifre olamaz.') + return url.href +} + +const shortHost = value => { try { return new URL(value).host } catch { return '' } } + +function describeCommand(action, args) { + switch (action) { + case 'browser_navigate': return `Sayfa açılıyor: ${shortHost(args.url) || String(args.url || '').slice(0, 60)}` + case 'browser_snapshot': return 'Sayfa okunuyor' + case 'browser_click': return `Tıklanıyor: ${String(args.ref || '').slice(0, 20)}` + case 'browser_type': return `Yazılıyor: ${String(args.ref || '').slice(0, 20)}` + case 'browser_scroll': return args.direction === 'up' ? 'Yukarı kaydırılıyor' : 'Aşağı kaydırılıyor' + case 'browser_back': return 'Geri gidiliyor' + case 'browser_press': return `Tuşa basılıyor: ${String(args.key || '').slice(0, 20)}` + case 'browser_get_images': return 'Görseller listeleniyor' + case 'browser_vision': return 'Ekran görüntüsü alınıyor' + default: return action + } +} + +export function createController({ chrome, setTimer = setTimeout, clearTimer = clearTimeout, now = () => Date.now(), random = () => crypto.randomUUID() }) { + let port = null, connected = false, registered = false, lastError = '', profile = '' + let enabled = true, instanceId = '', settingsLoaded = null, reconnectTimer = null, backoff = 2000 + let nextId = 0, titles = new Map(), titlesAt = 0 + const pending = new Map() + const active = new Map() + const queues = new Map() + const debuggers = new Map() + let state = null + + // ── persistent pieces ──────────────────────────────────────────────── + async function settings() { + settingsLoaded ||= (async () => { + const saved = await chrome.storage.local.get(['controlEnabled', 'controllerInstance']) + enabled = saved.controlEnabled !== false + instanceId = saved.controllerInstance || random() + if (!saved.controllerInstance) await chrome.storage.local.set({ controllerInstance: instanceId }) + })() + return settingsLoaded + } + + async function load() { + if (state) return state + const saved = await chrome.storage.session.get('controlState') + state = { tabs: {}, groups: {}, shared: null, log: [], ...(saved.controlState || {}) } + return state + } + + async function save() { await chrome.storage.session.set({ controlState: state }) } + + // ── native connection ──────────────────────────────────────────────── + function request(action, params = {}) { + if (!port) return Promise.reject(new Error('IDE bağlantısı yok.')) + const id = `c${++nextId}` + return new Promise((resolve, reject) => { + const timer = setTimer(() => { pending.delete(id); reject(new Error('IDE yanıt vermedi.')) }, 30000) + pending.set(id, { resolve, reject, timer }) + try { port.postMessage({ id, action, params }) } + catch (error) { clearTimer(timer); pending.delete(id); reject(error) } + }) + } + + function scheduleReconnect() { + if (!enabled || reconnectTimer) return + reconnectTimer = setTimer(() => { reconnectTimer = null; void connect() }, backoff) + backoff = Math.min(backoff * 2, 60000) + chrome.alarms?.create('aiturk-reconnect', { periodInMinutes: 1 }) + } + + async function connect() { + await settings() + if (!enabled || port) return + const current = chrome.runtime.connectNative(HOST) + port = current + current.onMessage.addListener(frame => onFrame(current, frame)) + current.onDisconnect.addListener(() => { + if (port !== current) return + port = null; connected = false; registered = false + for (const item of pending.values()) { clearTimer(item.timer); item.reject(new Error('IDE bağlantısı kapandı.')) } + pending.clear() + abortAll('IDE bağlantısı kapandı.') + lastError ||= 'AITURK IDE kapalı ya da Chrome bağlantısı kurulmamış.' + broadcast() + scheduleReconnect() + }) + try { + const status = await request('status') + profile = status?.profile || '' + connected = true; lastError = '' + await register() + backoff = 2000 + chrome.alarms?.clear('aiturk-reconnect') + } catch (error) { + lastError = error.message + } + broadcast() + } + + async function register() { + if (!port || !enabled) return + const result = await request('browser.register', { enabled: true, capabilities: [...CAPABILITIES], + controllerId: `aiturk-chrome-${instanceId}`, browserProfileId: instanceId }) + registered = Boolean(result?.registered) + } + + function onFrame(source, frame) { + if (source !== port || !frame || typeof frame !== 'object') return + if (frame.id !== undefined) { + const item = pending.get(frame.id) + if (!item) return + pending.delete(frame.id); clearTimer(item.timer) + frame.error ? item.reject(new Error(frame.error)) : item.resolve(frame.result) + return + } + if (frame.event === 'browser.command') void runCommand(frame) + else if (frame.event === 'browser.cancel') cancelCommand(frame.commandId, 'Ajan bu adımı iptal etti.') + else if (frame.event === 'disconnected') { + // The IDE restarted its backend; the native host is still here. + registered = false; connected = false + abortAll('IDE yeniden bağlanıyor.') + broadcast() + setTimer(() => { void reconnectBackend() }, 2000) + } else if (frame.event === 'error') { lastError = String(frame.message || '').slice(0, 300); broadcast() } + } + + async function reconnectBackend() { + if (!port || !enabled) return + try { + const status = await request('status') + profile = status?.profile || '' + connected = true; lastError = '' + await register() + } catch (error) { lastError = error.message; setTimer(() => { void reconnectBackend() }, 10000) } + broadcast() + } + + // ── commands ───────────────────────────────────────────────────────── + async function runCommand(frame) { + const { commandId, action, sessionId } = frame + const args = frame.arguments && typeof frame.arguments === 'object' ? frame.arguments : {} + const record = { commandId, cancelled: false, reason: '' } + active.set(commandId, record) + const entry = await logEntry({ at: now(), sessionId, action, text: describeCommand(action, args), status: 'running' }) + let reply + try { + if (!CAPABILITIES.includes(action)) throw new ToolError('Chrome bu işlemi desteklemiyor.') + const result = await withBudget(execute(action, args, sessionId, record), COMMAND_BUDGET_MS) + reply = { commandId, ok: true, result } + entry.status = result?.success === false ? 'failed' : 'ok' + if (result?.success === false) entry.note = String(result.error || '').slice(0, 200) + } catch (error) { + const message = String(error?.message || error).slice(0, 2000) + // Expected refusals reach the model as a normal tool result it can act on. + reply = error instanceof ToolError ? { commandId, ok: true, result: { success: false, error: message } } + : { commandId, ok: false, error: message } + entry.status = 'failed'; entry.note = message.slice(0, 200) + } + if (record.cancelled) { entry.status = 'cancelled'; entry.note = record.reason } + active.delete(commandId) + await save().catch(() => {}) + broadcast() + if (record.cancelled) return + try { await request('browser.result', reply) } catch {} + } + + function cancelCommand(commandId, reason) { + const record = active.get(commandId) + if (record) { record.cancelled = true; record.reason = reason } + } + + function abortAll(reason) { for (const record of active.values()) { record.cancelled = true; record.reason = reason } } + + function guard(record) { if (record?.cancelled) throw new ToolError(record.reason || 'Durduruldu.') } + + async function withBudget(promise, ms) { + let timer + const timeout = new Promise((_, reject) => { timer = setTimer(() => reject(new ToolError('Tarayıcı adımı zaman aşımına uğradı.')), ms) }) + try { return await Promise.race([promise, timeout]) } finally { clearTimer(timer) } + } + + // One step at a time per tab. The stored tail never rejects: the caller + // handles the step's own error, the queue only orders the steps. + function serial(tabId, task) { + const next = (queues.get(tabId) || Promise.resolve()).then(task) + const tail = next.catch(() => {}).then(() => { if (queues.get(tabId) === tail) queues.delete(tabId) }) + queues.set(tabId, tail) + return next + } + + async function execute(action, args, sessionId, record) { + if (action === 'browser_navigate') { + const url = checkUrl(args.url) + const tab = await agentTab(sessionId, true) + return serial(tab.id, async () => { + guard(record) + await chrome.tabs.update(tab.id, { url }) + const loaded = await waitForLoad(tab.id, LOAD_WAIT_MS) + guard(record) + const snap = await inject(tab.id, snapshotPage, [{ full: false }]) + return { success: true, url: snap.url, title: snap.title, snapshot: snap.snapshot, element_count: snap.element_count, + ...(loaded ? {} : { note: 'Sayfa yüklenmeye devam ediyor; gerekirse browser_snapshot ile tekrar bakın.' }) } + }) + } + const tab = await agentTab(sessionId, false) + return serial(tab.id, async () => { + guard(record) + const current = await chrome.tabs.get(tab.id) + const address = current.pendingUrl || current.url || '' + if (address === 'about:blank' || !address) { + if (action === 'browser_snapshot') return { success: true, snapshot: 'Boş sekme. Önce browser_navigate ile bir sayfa açın.', element_count: 0, url: 'about:blank', title: '' } + throw new ToolError('Sekme boş. Önce browser_navigate ile bir sayfa açın.') + } + checkUrl(address) + switch (action) { + case 'browser_snapshot': { + const snap = await inject(tab.id, snapshotPage, [{ full: args.full === true }]) + return { success: true, ...snap } + } + case 'browser_click': return click(tab, args.ref, record) + case 'browser_type': return type(tab, args.ref, typeof args.text === 'string' ? args.text : String(args.text ?? ''), record) + case 'browser_scroll': { + const result = await inject(tab.id, scrollPage, [args.direction === 'up' ? 'up' : 'down']) + return { success: true, direction: args.direction === 'up' ? 'up' : 'down', ...result } + } + case 'browser_back': { + try { await chrome.tabs.goBack(tab.id) } catch { throw new ToolError('Geri gidilecek sayfa yok.') } + await waitForLoad(tab.id, LOAD_WAIT_MS) + const after = await chrome.tabs.get(tab.id) + return { success: true, url: after.url, title: after.title } + } + case 'browser_press': return press(tab, args.key) + case 'browser_get_images': return { success: true, ...(await inject(tab.id, listImages, [])) } + case 'browser_vision': return vision(tab, args.annotate === true) + } + throw new ToolError('Chrome bu işlemi desteklemiyor.') + }) + } + + async function click(tab, ref, record) { + const located = await inject(tab.id, locateRef, [ref]) + if (located.error) throw new ToolError(located.error) + if (located.disabled) throw new ToolError(`${ref} şu an devre dışı.`) + guard(record) + let trusted = true + try { + const send = await debuggerFor(tab.id) + // No mouseMoved first: on a background tab Chrome holds a hover move + // for 5 s waiting for a frame, while press/release go straight through. + for (const type of ['mousePressed', 'mouseReleased']) { + await send('Input.dispatchMouseEvent', { type, x: located.x, y: located.y, button: 'left', clickCount: 1 }) + } + } catch { + trusted = false + const result = await inject(tab.id, clickWithEvents, [ref]) + if (result.error) throw new ToolError(result.error) + } + await settle(tab.id) + const after = await chrome.tabs.get(tab.id) + return { success: true, clicked: ref, element: located.name, url: after.url, title: after.title, + ...(located.covered ? { warning: `Tıklanan noktada başka bir öğe vardı: "${located.covered}". Gerekirse önce onu kapatın.` } : {}), + ...(trusted ? {} : { note: 'Gerçek fare tıklaması yerine sayfa olayı kullanıldı.' }) } + } + + async function type(tab, ref, text, record) { + const prepared = await inject(tab.id, prepareTyping, [ref]) + if (prepared.error) { + if (prepared.handoff) notify('AITURK senin girmeni bekliyor', prepared.error) + throw new ToolError(prepared.error) + } + guard(record) + if (prepared.select) { + const result = await inject(tab.id, typeWithEvents, [ref, text]) + if (result.error) throw new ToolError(result.error) + return { success: true, typed: ref, value: result.value } + } + try { + const send = await debuggerFor(tab.id) + if (text) await send('Input.insertText', { text }) + else for (const type of ['rawKeyDown', 'keyUp']) await send('Input.dispatchKeyEvent', { type, key: 'Delete', code: 'Delete', windowsVirtualKeyCode: 46 }) + } catch { + const result = await inject(tab.id, typeWithEvents, [ref, text]) + if (result.error) throw new ToolError(result.error) + } + const value = await inject(tab.id, readValue, [ref]) + return { success: true, typed: ref, value: value.value ?? '' } + } + + async function press(tab, key) { + const description = parseKey(key) + try { + const send = await debuggerFor(tab.id) + await send('Input.dispatchKeyEvent', { type: description.text ? 'keyDown' : 'rawKeyDown', ...description, + unmodifiedText: description.text }) + await send('Input.dispatchKeyEvent', { type: 'keyUp', key: description.key, code: description.code, + windowsVirtualKeyCode: description.windowsVirtualKeyCode, modifiers: description.modifiers }) + } catch { + throw new ToolError('Tuşa basılamadı: Chrome kontrol izni alınamadı (başka bir geliştirici aracı açık olabilir).') + } + await settle(tab.id) + const after = await chrome.tabs.get(tab.id) + return { success: true, pressed: String(key), url: after.url, title: after.title } + } + + async function vision(tab, annotate) { + let annotations = [] + if (annotate) { + await inject(tab.id, snapshotPage, [{ full: false }]) + annotations = (await inject(tab.id, annotateRefs, [true])).annotations || [] + } + try { + let data = '' + try { + const send = await debuggerFor(tab.id) + // A background tab paints on demand for the capture (about 1-3 s). + for (const options of [{ quality: 65, budget: 15000 }, { quality: 40, budget: 5000 }, { quality: 40, scale: 0.6, budget: 5000 }]) { + let clip + if (options.scale) { + const metrics = await send('Page.getLayoutMetrics', {}) + const view = metrics.cssVisualViewport || metrics.layoutViewport + clip = { x: view.pageX || 0, y: view.pageY || 0, width: view.clientWidth, height: view.clientHeight, scale: options.scale } + } + const shot = await withBudget(send('Page.captureScreenshot', { format: 'jpeg', quality: options.quality, ...(clip ? { clip } : {}) }), options.budget) + data = shot?.data || '' + if (data && data.length <= SCREENSHOT_LIMIT) break + } + } catch { + const current = await chrome.tabs.get(tab.id) + if (!current.active) throw new ToolError('Ekran görüntüsü alınamadı: ajanın sekmesi görünür değil ve Chrome kontrol izni alınamadı.') + const url = await chrome.tabs.captureVisibleTab(current.windowId, { format: 'jpeg', quality: 55 }) + data = String(url || '').replace(/^data:image\/jpeg;base64,/, '') + } + if (!data || data.length > SCREENSHOT_LIMIT) throw new ToolError('Ekran görüntüsü çok büyük ya da boş geldi.') + const after = await chrome.tabs.get(tab.id) + return { success: true, screenshot: data, mime: 'image/jpeg', url: after.url, title: after.title, + ...(annotate ? { annotations } : {}) } + } finally { + if (annotate) await inject(tab.id, annotateRefs, [false]).catch(() => {}) + } + } + + // ── tabs ───────────────────────────────────────────────────────────── + async function agentTab(sessionId, create) { + const current = await load() + const key = sessionId || 'default' + const known = current.tabs[key] + if (known) { + try { return await chrome.tabs.get(known) } catch { delete current.tabs[key] } + } + if (current.shared && (!current.shared.sessionId || current.shared.sessionId === key)) { + try { + const tab = await chrome.tabs.get(current.shared.tabId) + current.tabs[key] = tab.id + current.shared = null + await save() + return tab + } catch { current.shared = null } + } + if (!create) throw new ToolError('Bu konuşma için açık bir tarayıcı sekmesi yok. Önce browser_navigate ile bir sayfa açın.') + const window = await chrome.windows.getLastFocused({ windowTypes: ['normal'] }).catch(() => null) + const tab = await chrome.tabs.create({ url: 'about:blank', active: false, ...(window?.id ? { windowId: window.id } : {}) }) + current.tabs[key] = tab.id + await save() + await groupTab(tab, key).catch(() => {}) + return tab + } + + async function sessionTitle(sessionId) { + if (now() - titlesAt > 60000 && port) { + titlesAt = now() + try { + const data = await request('sessions.list') + titles = new Map((data?.sessions || []).map(row => [row.id, row.title])) + } catch {} + } + return titles.get(sessionId) || '' + } + + async function groupTab(tab, key) { + const current = await load() + let groupId = current.groups[key] + if (groupId !== undefined) { + try { + const group = await chrome.tabGroups.get(groupId) + if (group.windowId !== tab.windowId) groupId = undefined + } catch { groupId = undefined } + } + groupId = await chrome.tabs.group({ tabIds: [tab.id], ...(groupId === undefined ? { createProperties: { windowId: tab.windowId } } : { groupId }) }) + current.groups[key] = groupId + await save() + const title = await sessionTitle(key) + await chrome.tabGroups.update(groupId, { title: `AITURK${title ? ` · ${title.slice(0, 28)}` : ''}`, color: 'cyan' }) + } + + function waitForLoad(tabId, ms) { + return new Promise(resolve => { + let done = false + const finish = value => { + if (done) return + done = true; clearTimer(timer) + chrome.tabs.onUpdated.removeListener(listener) + resolve(value) + } + const listener = (id, change) => { if (id === tabId && change.status === 'complete') finish(true) } + const timer = setTimer(() => finish(false), ms) + chrome.tabs.onUpdated.addListener(listener) + // A page that finished before the listener existed. + // pendingUrl is set until the new document commits, so an old page's + // "complete" is never mistaken for the new one. + setTimer(() => { chrome.tabs.get(tabId).then(tab => { if (tab.status === 'complete' && !tab.pendingUrl) finish(true) }).catch(() => finish(false)) }, 250) + }) + } + + async function settle(tabId) { + await new Promise(resolve => setTimer(resolve, 400)) + const tab = await chrome.tabs.get(tabId).catch(() => null) + if (tab?.status === 'loading') await waitForLoad(tabId, 8000) + } + + async function inject(tabId, func, args) { + let results + try { + results = await chrome.scripting.executeScript({ target: { tabId }, func, args }) + } catch (error) { + throw new ToolError(`Sayfaya erişilemedi: ${String(error?.message || error).slice(0, 200)}`) + } + const value = results?.[0]?.result + if (value === undefined || value === null) throw new ToolError('Sayfa yanıt vermedi; yükleniyor olabilir.') + return value + } + + async function debuggerFor(tabId) { + let entry = debuggers.get(tabId) + if (!entry) { + await chrome.debugger.attach({ tabId }, '1.3') + entry = { timer: null } + debuggers.set(tabId, entry) + } + clearTimer(entry.timer) + entry.timer = setTimer(() => { void detachDebugger(tabId) }, DEBUGGER_IDLE_MS) + return (method, params) => chrome.debugger.sendCommand({ tabId }, method, params) + } + + async function detachDebugger(tabId) { + const entry = debuggers.get(tabId) + if (!entry) return + clearTimer(entry.timer) + debuggers.delete(tabId) + await chrome.debugger.detach({ tabId }).catch(() => {}) + } + + // ── state for the panel ───────────────────────────────────────────── + async function logEntry(entry) { + const current = await load() + current.log.unshift(entry) + current.log.length = Math.min(current.log.length, LOG_SIZE) + await save().catch(() => {}) + broadcast() + return entry + } + + function notify(title, message) { + chrome.notifications?.create({ type: 'basic', iconUrl: 'icon128.png', title, message: String(message).slice(0, 200), priority: 1 }) + ?.catch?.(() => {}) + } + + async function snapshotState() { + const current = await load() + const tabs = [] + for (const [sessionId, tabId] of Object.entries(current.tabs)) { + try { + const tab = await chrome.tabs.get(tabId) + tabs.push({ sessionId, tabId, title: (tab.title || '').slice(0, 120), url: shortHost(tab.url) }) + } catch { delete current.tabs[sessionId] } + } + return { enabled, connected, registered, error: lastError, profile, running: active.size, + log: current.log.slice(0, 15), tabs, shared: current.shared } + } + + function broadcast() { + chrome.action?.setBadgeText?.({ text: active.size ? '●' : '' })?.catch?.(() => {}) + void snapshotState().then(view => chrome.runtime.sendMessage({ type: 'control.state', state: view })).catch(() => {}) + } + + async function setEnabled(value) { + await settings() + enabled = value === true + await chrome.storage.local.set({ controlEnabled: enabled }) + if (enabled) { + backoff = 2000 + if (port && !registered) { try { await register() } catch (error) { lastError = error.message } } + else await connect() + } else { + abortAll('Kullanıcı ajanın tarayıcı kullanımını kapattı.') + for (const tabId of [...debuggers.keys()]) await detachDebugger(tabId) + if (port) { try { await request('browser.register', { enabled: false }) } catch {} } + registered = false + const previous = port + port = null; connected = false + previous?.disconnect() + chrome.alarms?.clear('aiturk-reconnect') + } + broadcast() + return snapshotState() + } + + async function shareTab(tab, sessionId) { + if (!tab?.id) throw new ToolError('Paylaşılacak sekme bulunamadı.') + checkUrl(tab.url || '') + const current = await load() + if (sessionId) current.tabs[sessionId] = tab.id + else current.shared = { tabId: tab.id, sessionId: null, title: (tab.title || '').slice(0, 120) } + await save() + broadcast() + return snapshotState() + } + + async function releaseTab(sessionId) { + const current = await load() + const ids = sessionId ? [current.tabs[sessionId]] : Object.values(current.tabs) + if (sessionId) delete current.tabs[sessionId] + else { current.tabs = {}; current.shared = null } + for (const tabId of ids) if (tabId) await detachDebugger(tabId) + await save() + broadcast() + return snapshotState() + } + + async function onMessage(message, sender) { + switch (message?.type) { + case 'control.get': + await settings() + if (enabled && !port) void connect() + return snapshotState() + case 'control.set': return setEnabled(message.enabled === true) + case 'control.stop': return setEnabled(false) + case 'control.share': { + const [tab] = await chrome.tabs.query({ active: true, lastFocusedWindow: true }) + return shareTab(tab, typeof message.sessionId === 'string' ? message.sessionId : null) + } + case 'control.release': return releaseTab(typeof message.sessionId === 'string' ? message.sessionId : null) + } + return undefined + } + + function onDebuggerDetach(source, reason) { + const entry = debuggers.get(source?.tabId) + if (!entry) return + clearTimer(entry.timer) + debuggers.delete(source.tabId) + if (reason === 'canceled_by_user') { + notify('AITURK durduruldu', 'Chrome uyarısındaki İptal ile ajanın tarayıcı kullanımı kapatıldı. Panelden tekrar açabilirsiniz.') + void setEnabled(false) + } + } + + async function onTabRemoved(tabId) { + const current = await load() + let changed = false + for (const [key, value] of Object.entries(current.tabs)) if (value === tabId) { delete current.tabs[key]; changed = true } + if (current.shared?.tabId === tabId) { current.shared = null; changed = true } + debuggers.delete(tabId) + if (changed) { await save(); broadcast() } + } + + return { connect, onMessage, onDebuggerDetach, onTabRemoved, setEnabled, shareTab, releaseTab, snapshotState, + onAlarm: name => { if (name === 'aiturk-reconnect') void connect() }, + // exposed for tests + _execute: execute, _runCommand: runCommand, _state: () => ({ port, connected, registered, enabled, active }) } +} diff --git a/apps/chrome/extension/icon128.png b/apps/chrome/extension/icon128.png new file mode 100644 index 0000000000000000000000000000000000000000..d0b6620246a44faf71c6f6e5e301ad3908e6c982 GIT binary patch literal 4964 zcmV-q6PxUbP)o3b;p1AzNNcow2o|Rv`ZdoB#mZ~jR{@=*@hqtn8jFwW#F4X1PY~G#qKK7^Bf1v zRz*bxsY>xBA(f;?DPm!4P%1VSAqW#}8H`7>NXU}4ueQ)E{g!+3pIU3>t7Ku~HA$S=_9$AtE?c%BbHceh1nrvc>ZE5L>g z7F)K&g8%#tSh-JwJcRA6LMsSD2t$Bq)JiC&CZTLobqOLe=5$-wHaIpIfiUO=^+TfJ zAK`uX*afn=DhiO#+jw5crcD{FUtNb|eGk`NOQ8T!6sAId3zfmLY%&?}{g()$ zhxyxYY$u=BbahFR*gz#X^%~DJ0CYW)$I@LW4Za@&R&3tW_0}nZ01>WhBL*VVMc3E8 z3uHGT1t|HH{H7V`8UIE$^8p5j30Fk0q_~sX<Qe$u5QNm_<}qY$;?c+c z1+dw^T`sKv-Q5LN%%g`Md&bq_7j$ZNc{=l(N$O zs|l?3o`NMDI#+c<*vl@P9sGhZ?k>7;jY{Nid^nvsp+p- zE1)nYqONWlQQ;`h{b~nb7iro}n_Onszk=i381v5Q>918Q;5dpP_!FlG*RXlBuK}T< zejV9t6TY7czt+=2-xspkCK~G3oeKh$+JM%AQYPJ{npXl7%X&j8{mVk32isXj5SY{g zObIuj34^_mY0;R%O=!J@Fig$=RB;1gC}_O|Q@BY3w_!UPAvOO~2OBiljs|YiXm!h^ z>Vwl&g;szPb$p8knh|qRsxVzO8wV)F05oev8i@>PZQm3Fg(3qYjT)sI;)ab>0j81@ z5CNqcblR>pW!(TMmD8#4r?wKLdH)n&sRB&#RrAH3w$_s@vlmPayUChYfN*edB3;wx zmg9os)NE_JU8_Ksh~U~b55IRkb&gZE3K$WzQf%8j!1H_epj@{mEkR8xfYur_Jj}YL z6@2qA-#@9}_3kB0dCxx%)GSC+vo1i4;gjua35_8%rtDZi5I}L;!o@6}JD2?@PM{s9 z=B$9ONt81TnLWFKdsnW))|!lEVQXF1<7f*7P_JIZTsjBf!( z0Vd!BD^@alRs&HO)RqJxsxfedw64HguY`(dHL9pvQoy98;}?!Ocj6%fPO72 zG4vb7zWkT_UL_nFV&{>=4ElaCjZ6}20Cz2GV!`z{5CuWiy8&Gb(C^Z@jjUU;ED`iu zy13v!cfA6Z=Fq8A{A&Nc_-+d`ezsu@b&kVFSFHv=sCqY`YXbURy@pHz=vM$sYlZ@! zZF>g5%|ICNL~l=`IACn|%*NIZaO|pQ2&-BFB4bdQY%w)U;KxfR{8#(;bLiv=Y|8@M z;b*)1DTE=m*5fLH6@TU(3m37faemb^gu1$+U!R*7E@ExtykyXC{O1$9UO~lZo^~Ae z9Y4nN2VYC%21<%&Yt8*Nz!2)HhJK@vjR~P&p;#hb2t%IU-H$j9Q3)^#1Ge_=O6&$o z0`FYYj?Pv$L#V46`pr(8UOa{4K)+*8_VWi1u(fU3k0*=E|1ATb5@ZdOcmhjL%+-BHuCNz zOA|rACN}>sA3ee=hYz7H(cL2=VA=d6jsucJ^(BVzy1IJSFOM0*Dsuz68lm3}Ev;m% zWL3WqDg+8QN?l0dvAhH#qe@{#7%P-UQyt~TiR!PfS&!J41;_Yw{djq9I{G2ps9 z|Jr``zw!ELs+X7{Z0i8Kx)?%TZP4$>TiOzz&eCVL_U;198n>2H0Nb({IeV6$4fK}< zG6`_|qQxw4Y{UfBLlEg|f`0G5aXE`;&OArrB$)8A_w;Fgdtg7x&0rGb>H;XOc%rwb zm}MjsWMmAEWpV%N)~aeH*VPF9N*U(==ezp}hK6vg;!XE@oCiQv39|ts$mH2YKZ>Bg3rA-uVz-Zc(Tf&Y`^usdkF4+Up(zW$_O|k z=Fgm!7!)cgimf%DXl>)gKMkN%w&I&e>}r61iK~7m0vx?-@zN`kfs|1w^6{2dzV_eS z2qJ@0R5UoGs{#6LYMjTrmMldkFz_J~Au?t{Pr2z_$?7!CZs3lkHzHNgL#V3-`gJX< z$ox&fg;-io=$S|)e_ktj6$FC1I-uY7D>m@qTEr{>A6~hd=`&|mJVU4}9s0dx(GuF{ z&MOvbRDlAN3RLG#o5p*aR-k;pq8UP6@z8I4`l|_E9otGSq(R_>q2JuJ8Qhn!>bIh- zSIiK;uW1F>#NP**hc+q9fTGm=5S7exM7^eHbRC^3XHG8yh|UR87hsRERLrA;}| z?*)w~@VLno&{(7!UgOeLlQ$H8Zo>#FM^D?BA~A$9Vfc z{QZ@KVB_gy4YOvm_u(%xBa?~eh$nLc8QbE%)h&GY&;fJ>-Z`Kv68g2zLBFa@0OqOw zKJb0qOa|pRlYeX*JD20s>!|C7F5VCmu{kSp{6C}VhHS5Ns^h6p#$n@`(o$7+sn%R zOC37fb`MY(Jd15vR}_3PB52#;z{%t6cx``KZlLskxPMJ6PJDksssNXPe&4=mF>P}j z6LtKK-DCx9?dt)pt_t#2oFfpGHAi49L)bj$I&NLEwCDyZ2pZOv2mN-G1^t%Bz8xJr z%m3_q^@^MS^MXvr;pyFd1W{Dh9Yp6C!VZW`MUg|glAzzZ=`*-*^_sG)a0GZ}&mPX6 zIfG@}SH%L98c25^J<1D*55?bY%3=sVvZ{sp=`$*-m0VX0c3vp(o~Gq2jG^D8!BT1L z+Y@~~qkq4uRBzz(lhRZ%DUTtXKYcoPH!VjMD#r~}#9NdYvro0(R2K9r2$t5oF*3rl zd;c^#_vh+j6b?@g^u==obk#G2x>BIu>*vhj{bfSGktuHD_INmU;Eu}dETl;olDNVM__ti@A*SFq|u8X?5)X@Gu>k>8OzfT`vHh>4irXplJ2jp+f>O`mI0B>!Q2Wc^ID%5o z;CUlD)`~3{qYz2mKR-guRe?vB-z&wWMog$yNP)gLf(y+V&Y^1Og*KNV~ zBb2hzovVq{AK|(dzTbn&+ysPpo&cj!ztGd8@H`)jhY$g+Dt*FPx{{uKtppLUcnHt) z>FF7roPEyEZ`l&j-EEP7dQ8@2uFfV1y2wAigYIsNEn6ZkS`pOQ zX~^eo9(~+naOmgM4eiKrca~K@@IgdW>kj{D$ayc83 zAl=ofG5Z0zoJ|n);18|m%isRvU5B#))LP#Szc1gumMDxkazB3#!-3`C}j zuCIF+$nFwre4uChc0oh|8t<{^c;_9@q19Tl+4&fwMze%e5w1P73lWxOk;`eq@Fjxi z0UrIvKatOCo_h}Nx=WJIG)}hic^l6Q*|aIcoN1p$>o4JE8VLdj0uvKSI^_|rs?dvj z>^K_70l#nrW4^`7vpZ>V+hHP)x(KYs&O?$aO-VLPkP3W5;A5Fi>&D~r}4 zwGibZcp_udHVfMZ#|9%12ECwuNHqK-yzd^nKsHxR0ZKpD-L2{Dj1?j8VuyEQLgxw!W9fNgnudpvLG iJeSgCt4olj~6$5CtJgQZRrSMYTwr z0Jw%iW?;7YZZt=nfND`OGi0-dh(MZhuw^e_Zr)+()?LnYc0r|rtBXSf%uKg3bRtkG zGt%47;DHW~6$)G*Itgi-fj64?t)5^8KH|oy(|q{(gJ0|GjP?z1JD=xI93vvsH>CB+ zDnh7K=;aOo&Lg?3(iTBLLO z5r>{k@$k(n+HyHY`v$gc!&h0!hDu6p`+2?mjpvICto&XDRc1ebX00000NkvXXu0mjf3UVE?q`mg_1 z0OWFpO`AO1x6h+B^)@1di1Z?20mXxvpp-+YvnZA4cD$cGdoJBwQ&krK;N5kk)1PAO zT$}^%nNl+btOf6`P%6E}$9sp#<;=wHK74PGrpzI{hr~I+pp=@@07L}vMAl*qjNe(q zuHAXXjvaGI$tjHOBuP9<86KvIc%sPQU4d8)Xsm1_o&IkY2T;n8B%XA-6D!*^u#UtD zdWw&{nrWp-oB-=ANDtntM$oPRqxC&wUZkAny6QqtfLwYkO6h-tCpR`W8I1txG-y+P zsXHCoHkLkxc+d2fRz6v~0jmunf>Mf|hreUAa2*{*1gSrWh5|53;YuaeE`NqMo_zYA zZ=*MEu=DFYI+bcDfQF7gh=4V`Hn0pA00QobgIeg)K#b33UDLW=@JSr=RM%r-vm+NrDUBET1vu|%YHMf1(!R?0fsvo^V`I2mH#vB6q)c$|%7VqTcTB@OS63Qp@_^QwFka-v zMN639-a#l+?&8iFbUyb2NZ`@7_6kmBw6yS2??S#BIgV*=!PO{BTOakp>jO_9WiHT4 zv3J$#zjH+S6t-lS@#Tr1>LO)bEB8K0=zL^4t9uugn~?g)#;KLUD1`!6_VhA)`V4%M z)GSF`lX~NrHH((ek;&jgz-Y~z`C-mpzlMz>09+EYboN~KzdVQwf!0)t!9|NXaP$Z! zif~n^+18|9t$DRSTgl~_LV?eJJT{>j@9s_l-(MKzoo81tue}2o0-O4>9Qff_&AyH8J)a$DV7$cz0Yhakjs)l6DgIW+FvI-d1IJJ*bsX!DNMEpZv{O-l1R4U-qKF!e3 zReacr(GXdKh^t>;Hz=h=9Fa8`4L)pTXy~dYmox0zou^n_4_qOWu~<9xmlSIinT!Rl mP%N%z*Y5n}<@A5Np8pA2ToySH=AhvK0000N!dL|*nN{p zzI@+zG0)8N&hyMWBU~K){g$}VhbkX7ZS2G)KOx3%k|YVrZTc@_2oaP{haulckrKd~@y8-WMH|Ib zRWRgTqnWR&_zRzo|1n=EscYj52d;4=FK7(RXldblEAJUsynpf58=O9W9&xTLvh_#9 z09YgVks{w(ww%phy?5My&W()lvsd?ElLX%&0H$$DuizXH^)AK7NYQ(;7|6=cK1Lqw z?u8^*`nN&R5CBGmn58Ue@8GVvw_uGS?;MG-B;{vo49jnxLs!R5#5Bb;$R9Qg09i)g zZQUf+5@H14(==t&`|=7DvBt3Wwr=p*)B+G<#1xuX*V9{G97+^<{n$|spFLCaZ(ny0 zw$M}sdCdV>5j0J?eeOJZKG#lEMatWZ-yC?4eaHV@lP8OBoXxVix1ec?ZGd)~`bAM7 zcpmCmQko4_tT9AYcD#RpSB?zTCOWXLy9a`&5iBzGs;&=#=IdK|plfmUodEBgJi)0m zXL#-Chm3lUGp5S>2f7whxbFIrd%uJzW#^%TXp%sdvE$%@s(p!CO7ZpG zy$vZ8n0mM$vWy#N&g8xY3#-%-5f1$4WA>joj&TV%$L^s+`0_Gpq?F?KFI+@h+YEyD zb_akhCYDeC^>6Ma(Rw?e472IP9o(Q3o$aQxrMJUScs&> z%evRk6hO&3ST^)5t{-WPC91I;>5XglXj?e z0LWNdwhlfP21r^0mE_cBrN<>(Jdh=7q}i2r*I&o}#a9 zX$UoWQc2wP-All^Njp?imvxw3vJMphB2sdq){%F$EoW_&W)%%l;Lf@8=xpyGW|K2` zu3qa<895`uyQfd`<2^59lB8xtr93-{bNu|iRm^N|F3pxIiM6chTFkbOj$v{W-z&Ju zScj&Itb-A$&cbu=yv?uQ{1dcXSE^{&CJu~@uxjBKS=Y0Kpjyp`2fKUN{>MK|%Arn( zLdF;*EppHNG3!vFO;ly~&>?JV8@Z+?Tyt|x%Qv^+3QfFlaIh+P71drcyMv`S&q1@a zv`T%JIiOKl*WHVckt_tVa&qoJb&`LaI8L;dGz5H%H7yMRpE!R1&%+FljG(GyF;EPF zpvs!AZpi$EEBR(()*)qXdpjF?m*R|}P)<%~47)#g9~~XNtg5Oi)+L-7KFjO>I*K!f z#8?W>;f&$Cw=buqwUsbIhw3J3pjF{JJDx9ZTuO>6z$=G`AfK;sq~n1Q>)5#GC7vIA zkEo^8s0>iz5;RRGGz-F}ALxXReY}S>9j9oO2W$Vo!z}Ng$+Ano74k5_U7D*7;}w?k z#5!zCa;dnfUYQk8#Uu$Pmm60gb%M*HBT~f371u`<#Ye5XhE#WLNCcu54Z;xCfNEpP z1Xt&%DPj!>Lk5j|u?~>wbHITp)`7;o28qv*r5?ldIp8n|S?ZDajA8SW`v~Dh3QdNn ze#&<~s;AIo2;oIGKe^AKDg^f^#q<-LwdKQYrtC4RvzB7|3Bf&zs#pdGEl)pvmb>qG zmn7K$7((<@?#OBEpj*h&HEjLqUl<&;hJk@Vf4^nR_FWW"], + "background": { "service_worker": "background.js", "type": "module" }, + "action": { "default_title": "AITURK yan panelini aç", "default_icon": { "16": "icon16.png", "32": "icon32.png", "48": "icon48.png" } }, + "icons": { "16": "icon16.png", "32": "icon32.png", "48": "icon48.png", "128": "icon128.png" }, + "commands": { "_execute_action": { "suggested_key": { "default": "Alt+Shift+A" }, "description": "AITURK yan panelini aç" } }, "side_panel": { "default_path": "panel.html" }, "content_security_policy": { "extension_pages": "script-src 'self'; object-src 'none'; connect-src 'none'; base-uri 'none'" }, "homepage_url": "https://turkservis.online/indir" diff --git a/apps/chrome/extension/page-agent.js b/apps/chrome/extension/page-agent.js new file mode 100644 index 0000000..c14e6ce --- /dev/null +++ b/apps/chrome/extension/page-agent.js @@ -0,0 +1,341 @@ +// Functions injected into a page the IDE agent controls. Each one runs through +// chrome.scripting.executeScript in Chrome's isolated world, so every function +// must be self-contained (no imports, no outer variables). They share one +// global per document: the element map of the latest snapshot, which turns the +// model's "@e5" back into the element it saw. + +// Accessibility-style outline of the page with refs for interactive elements. +// full=false lists interactive elements and headings; full=true adds visible +// text in reading order. Same-origin iframes and open shadow roots are walked. +export function snapshotPage(options) { + const full = Boolean(options && options.full) + const limit = Math.max(2000, Math.min(Number(options && options.limit) || 15000, 40000)) + const refs = new Map() + const lines = [] + let counter = 0, length = 0, omitted = 0 + const ROLE_BY_TAG = { a: 'link', button: 'button', select: 'combobox', textarea: 'textbox', summary: 'button', + h1: 'heading', h2: 'heading', h3: 'heading', h4: 'heading', h5: 'heading', h6: 'heading', img: 'img', iframe: 'iframe' } + const INTERACTIVE = new Set(['button', 'link', 'textbox', 'searchbox', 'combobox', 'checkbox', 'radio', 'switch', 'tab', + 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem', 'listbox']) + const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'HEAD', 'META', 'LINK']) + const clean = (value, max = 100) => String(value || '').replace(/\s+/g, ' ').trim().slice(0, max) + const quote = value => JSON.stringify(value) + + function visible(element) { + const rect = element.getBoundingClientRect() + if (rect.width < 1 || rect.height < 1) return false + const style = element.ownerDocument.defaultView.getComputedStyle(element) + return style.visibility !== 'hidden' && style.display !== 'none' && style.opacity !== '0' + } + + function inputRole(element) { + const type = (element.getAttribute('type') || 'text').toLowerCase() + if (['button', 'submit', 'reset', 'image'].includes(type)) return 'button' + if (type === 'checkbox') return 'checkbox' + if (type === 'radio') return 'radio' + if (type === 'range') return 'slider' + if (type === 'number') return 'spinbutton' + if (type === 'search') return 'searchbox' + if (type === 'hidden') return '' + return 'textbox' + } + + function roleOf(element) { + const explicit = clean(element.getAttribute('role'), 40).split(' ')[0] + if (explicit) return explicit + const tag = element.tagName.toLowerCase() + if (tag === 'input') return inputRole(element) + if (tag === 'a') return element.hasAttribute('href') ? 'link' : '' + if (element.isContentEditable && element.getAttribute('contenteditable') !== null) return 'textbox' + return ROLE_BY_TAG[tag] || '' + } + + function nameOf(element, role) { + const doc = element.ownerDocument + const labelled = element.getAttribute('aria-labelledby') + if (labelled) { + const text = labelled.split(/\s+/).map(id => doc.getElementById(id)?.textContent || '').join(' ') + if (clean(text)) return clean(text) + } + const aria = element.getAttribute('aria-label') + if (clean(aria)) return clean(aria) + if (element.id && ['textbox', 'searchbox', 'combobox', 'checkbox', 'radio', 'slider', 'spinbutton', 'switch'].includes(role)) { + const label = Array.from(doc.getElementsByTagName('label')).find(candidate => candidate.htmlFor === element.id) + if (label && clean(label.textContent)) return clean(label.textContent) + } + const wrapping = element.closest('label') + if (wrapping && wrapping !== element && clean(wrapping.textContent)) return clean(wrapping.textContent) + if (element.tagName === 'IMG') return clean(element.getAttribute('alt')) + if (element.tagName === 'INPUT' && ['button', 'submit', 'reset'].includes((element.type || '').toLowerCase())) { + return clean(element.value || element.type) + } + const text = role === 'textbox' || role === 'searchbox' || role === 'combobox' ? '' : clean(element.innerText || element.textContent) + return text || clean(element.getAttribute('title')) || clean(element.getAttribute('placeholder')) + } + + function states(element, role) { + const out = [] + if (role === 'heading') out.push(`level=${element.tagName.match(/\d/)?.[0] || element.getAttribute('aria-level') || 2}`) + if (element.disabled || element.getAttribute('aria-disabled') === 'true') out.push('disabled') + if (element.checked || element.getAttribute('aria-checked') === 'true') out.push('checked') + if (element.getAttribute('aria-expanded') === 'true') out.push('expanded') + if (element.getAttribute('aria-selected') === 'true') out.push('selected') + if (element.required) out.push('required') + return out + } + + function valueOf(element, role) { + if (!['textbox', 'searchbox', 'combobox', 'spinbutton', 'slider'].includes(role)) return '' + const type = (element.getAttribute('type') || '').toLowerCase() + if (type === 'password') return element.value ? '••••' : '' + if (element.tagName === 'SELECT') return clean(element.selectedOptions?.[0]?.textContent, 80) + if (element.isContentEditable) return clean(element.innerText, 120) + return clean(element.value, 120) + } + + function push(line) { + if (length + line.length + 1 > limit) { omitted++; return } + lines.push(line) + length += line.length + 1 + } + + function describe(element, depth) { + const role = roleOf(element) + if (!role) return false + const interactive = INTERACTIVE.has(role) || (element.tagName === 'INPUT' && role) + if (!interactive && role !== 'heading' && !(full && (role === 'img' || role === 'iframe'))) return false + if (!visible(element)) return false + const name = nameOf(element, role) + const parts = [`${' '.repeat(depth)}- ${role}`] + if (name) parts.push(quote(name)) + const flags = states(element, role) + let ref = '' + if (interactive) { + ref = `e${++counter}` + refs.set(ref, new WeakRef(element)) + flags.push(`ref=${ref}`) + } + if (flags.length) parts.push(`[${flags.join('] [')}]`) + const value = valueOf(element, role) + if (value) parts.push(`(değer: ${quote(value)})`) + if ((element.getAttribute('type') || '').toLowerCase() === 'password') parts.push('(şifre alanı: ajan yazamaz)') + if (role === 'link' && full) { + try { parts.push(`→ ${new URL(element.getAttribute('href'), element.baseURI).href.slice(0, 160)}`) } catch {} + } + push(parts.join(' ')) + if (full && element.tagName === 'SELECT') { + Array.from(element.options).slice(0, 20).forEach(option => push(`${' '.repeat(depth + 1)}- option ${quote(clean(option.textContent))}${option.selected ? ' [selected]' : ''}`)) + } + return interactive + } + + function directText(element) { + let text = '' + for (const node of element.childNodes) if (node.nodeType === 3) text += node.textContent + return clean(text, 300) + } + + function walk(root, depth) { + for (const element of root.children || []) { + if (SKIP.has(element.tagName.toUpperCase())) continue + if (element.getAttribute('aria-hidden') === 'true' || element.hidden) continue + if (element.tagName === 'IFRAME') { + let inner = null + try { inner = element.contentDocument } catch {} + if (inner?.body && visible(element)) { push(`${' '.repeat(depth)}- iframe ${quote(clean(element.title || element.name || ''))}`); walk(inner.body, depth + 1) } + else if (visible(element)) push(`${' '.repeat(depth)}- iframe ${quote(clean(element.title || element.src, 80))} (farklı alan adı: içerik okunamaz)`) + continue + } + const interactive = describe(element, depth) + if (!interactive && full && visible(element)) { + const text = directText(element) + if (text && !roleOf(element)) push(`${' '.repeat(depth)}- text: ${text}`) + } + if (element.shadowRoot) walk(element.shadowRoot, depth) + // A described control's own children rarely add meaning (button labels). + if (!interactive || element.tagName === 'SELECT' || element.getAttribute('role') === 'listbox') walk(element, depth) + } + } + + walk(document.body || document.documentElement, 0) + globalThis.__aiturkRefs = refs + const header = [`Sayfa: ${clean(document.title, 200) || '(başlıksız)'}`, `Adres: ${location.href.slice(0, 300)}`] + if (!lines.length) lines.push('(Görünür etkileşimli öğe yok. Sayfa yükleniyor olabilir; browser_snapshot ile tekrar bakın ya da full=true deneyin.)') + if (omitted) lines.push(`… (${omitted} satır daha var; kısaltıldı. Sayfayı kaydırıp tekrar bakın.)`) + return { snapshot: header.concat(lines).join('\n'), element_count: counter, url: location.href, title: document.title, truncated: omitted > 0 } +} + +// Where to click/type for a ref, after scrolling it into view. Coordinates are +// top-level viewport CSS pixels (same-origin iframe offsets added). +export function locateRef(ref) { + const key = String(ref || '').trim().replace(/^@/, '').replace(/^ref=/, '').replace(/^\[|\]$/g, '') + const element = globalThis.__aiturkRefs?.get(key)?.deref() + if (!element || !element.isConnected) { + return { error: `Öğe bulunamadı (@${key}). Sayfa değişmiş olabilir; browser_snapshot ile yeni numaraları alın.` } + } + element.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }) + const rect = element.getBoundingClientRect() + let x = rect.left + rect.width / 2, y = rect.top + rect.height / 2 + let view = element.ownerDocument.defaultView + while (view && view !== window && view.frameElement) { + const frame = view.frameElement.getBoundingClientRect() + x += frame.left; y += frame.top + view = view.parent + } + const type = (element.getAttribute('type') || '').toLowerCase() + const autocomplete = (element.getAttribute('autocomplete') || '').toLowerCase() + let covered = '' + if (element.ownerDocument === document) { + const top = document.elementFromPoint(x, y) + if (top && top !== element && !element.contains(top) && !top.contains(element)) { + covered = String(top.innerText || top.getAttribute('aria-label') || top.tagName).replace(/\s+/g, ' ').trim().slice(0, 80) + } + } + return { + x: Math.round(x), y: Math.round(y), + tag: element.tagName.toLowerCase(), + name: String(element.getAttribute('aria-label') || element.innerText || element.value || element.getAttribute('title') || '').replace(/\s+/g, ' ').trim().slice(0, 80), + editable: element.isContentEditable || ['INPUT', 'TEXTAREA', 'SELECT'].includes(element.tagName), + password: type === 'password', + payment: /^cc-|^card|card-?number|cvc|cvv/.test(autocomplete) || /cvc|cvv|card.?number/i.test(element.name || ''), + disabled: Boolean(element.disabled || element.getAttribute('aria-disabled') === 'true'), + covered, + } +} + +// Focus a ref and select its current content so typed text replaces it. +// Refuses password and payment-card fields: those stay with the user. +export function prepareTyping(ref) { + const key = String(ref || '').trim().replace(/^@/, '').replace(/^ref=/, '').replace(/^[|]$/g, '') + const element = globalThis.__aiturkRefs?.get(key)?.deref() + if (!element || !element.isConnected) { + return { error: `Öğe bulunamadı (@${key}). Sayfa değişmiş olabilir; browser_snapshot ile yeni numaraları alın.` } + } + const type = (element.getAttribute('type') || '').toLowerCase() + const autocomplete = (element.getAttribute('autocomplete') || '').toLowerCase() + if (type === 'password' || autocomplete.includes('password')) { + return { error: 'Şifre alanına ajan yazamaz. Bu adımı kullanıcıya bırakın.', handoff: true } + } + if (/^cc-|card-?number|cvc|cvv/.test(autocomplete) || /cvc|cvv|card.?number/i.test(element.name || '')) { + return { error: 'Kart bilgisi alanına ajan yazamaz. Bu adımı kullanıcıya bırakın.', handoff: true } + } + const editable = element.isContentEditable || ['INPUT', 'TEXTAREA', 'SELECT'].includes(element.tagName) + if (!editable) return { error: `@${key} bir yazı alanı değil (${element.tagName.toLowerCase()}).` } + if (element.disabled || element.readOnly || element.getAttribute('aria-disabled') === 'true') { + return { error: `@${key} şu an devre dışı ya da salt okunur.` } + } + element.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }) + if (element.tagName === 'SELECT') return { select: true } + element.focus() + if (element.isContentEditable) { + const range = element.ownerDocument.createRange() + range.selectNodeContents(element) + const selection = element.ownerDocument.getSelection() + selection.removeAllRanges(); selection.addRange(range) + } else if (typeof element.select === 'function' && element.tagName !== 'SELECT') { + element.select() + } + return { focused: element.ownerDocument.activeElement === element } +} + +// Script fallback when trusted input is unavailable: replace the value and fire +// the events frameworks listen to. +export function typeWithEvents(ref, text) { + const key = String(ref || '').trim().replace(/^@/, '').replace(/^ref=/, '') + const element = globalThis.__aiturkRefs?.get(key)?.deref() + if (!element) return { error: `Öğe bulunamadı (@${key}).` } + if (element.tagName === 'SELECT') { + const wanted = String(text).trim().toLocaleLowerCase() + const option = Array.from(element.options).find(item => item.textContent.trim().toLocaleLowerCase() === wanted || + item.value.toLocaleLowerCase() === wanted) || Array.from(element.options).find(item => item.textContent.trim().toLocaleLowerCase().includes(wanted)) + if (!option) return { error: `Listede "${String(text).slice(0, 80)}" seçeneği yok.` } + element.value = option.value + } else if (element.isContentEditable) { + element.textContent = text + } else { + const proto = element.tagName === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype + const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set + setter ? setter.call(element, text) : (element.value = text) + } + element.dispatchEvent(new Event('input', { bubbles: true })) + element.dispatchEvent(new Event('change', { bubbles: true })) + return { value: element.isContentEditable ? element.textContent : element.value } +} + +// Read back what a field holds after typing (password fields never echoed). +export function readValue(ref) { + const key = String(ref || '').trim().replace(/^@/, '').replace(/^ref=/, '') + const element = globalThis.__aiturkRefs?.get(key)?.deref() + if (!element) return { error: `Öğe bulunamadı (@${key}).` } + return { value: String(element.isContentEditable ? element.innerText : element.value ?? '').slice(0, 2000) } +} + +export function clickWithEvents(ref) { + const key = String(ref || '').trim().replace(/^@/, '').replace(/^ref=/, '') + const element = globalThis.__aiturkRefs?.get(key)?.deref() + if (!element) return { error: `Öğe bulunamadı (@${key}).` } + element.click() + return { clicked: true } +} + +// Scroll the page, or the scrollable area under the viewport centre when the +// window itself does not move (single-page apps). +export function scrollPage(direction) { + const sign = direction === 'up' ? -1 : 1 + const amount = Math.round(window.innerHeight * 0.8) * sign + const before = window.scrollY + window.scrollBy({ top: amount, behavior: 'instant' }) + if (window.scrollY !== before) { + return { scrolled: true, atEnd: sign > 0 ? window.innerHeight + window.scrollY >= document.documentElement.scrollHeight - 2 : window.scrollY <= 0 } + } + let element = document.elementFromPoint(window.innerWidth / 2, window.innerHeight / 2) + while (element && element !== document.body) { + const style = getComputedStyle(element) + if (/(auto|scroll)/.test(style.overflowY) && element.scrollHeight > element.clientHeight) { + const previous = element.scrollTop + element.scrollBy({ top: amount, behavior: 'instant' }) + if (element.scrollTop !== previous) { + return { scrolled: true, atEnd: sign > 0 ? element.scrollTop + element.clientHeight >= element.scrollHeight - 2 : element.scrollTop <= 0 } + } + } + element = element.parentElement + } + return { scrolled: false, atEnd: true } +} + +export function listImages() { + const images = [] + for (const image of document.images) { + if (images.length >= 100) break + const rect = image.getBoundingClientRect() + if (rect.width < 8 || rect.height < 8) continue + const src = image.currentSrc || image.src || '' + images.push({ src: src.startsWith('data:') ? `${src.slice(0, 60)}…` : src.slice(0, 500), + alt: String(image.alt || '').slice(0, 200), width: Math.round(rect.width), height: Math.round(rect.height) }) + } + return { images, count: images.length, url: location.href, title: document.title } +} + +// Numbered labels over the refs of the latest snapshot, for annotated +// screenshots; call again with show=false to remove them. +export function annotateRefs(show) { + document.getElementById('__aiturk_labels')?.remove() + if (!show) return { removed: true } + const layer = document.createElement('div') + layer.id = '__aiturk_labels' + layer.style.cssText = 'position:fixed;inset:0;pointer-events:none;z-index:2147483647' + const annotations = [] + for (const [ref, weak] of globalThis.__aiturkRefs || []) { + const element = weak.deref() + if (!element || element.ownerDocument !== document) continue + const rect = element.getBoundingClientRect() + if (rect.bottom < 0 || rect.top > innerHeight || rect.right < 0 || rect.left > innerWidth) continue + const label = document.createElement('span') + label.textContent = ref.slice(1) + label.style.cssText = `position:fixed;left:${Math.max(0, rect.left)}px;top:${Math.max(0, rect.top)}px;background:#007f79;color:#fff;font:600 11px/14px system-ui;padding:0 3px;border-radius:3px` + layer.append(label) + annotations.push({ ref: `@${ref}`, label: `[${ref.slice(1)}]` }) + } + document.documentElement.append(layer) + return { annotations } +} diff --git a/apps/chrome/extension/panel.css b/apps/chrome/extension/panel.css index 6d74720..5465f07 100644 --- a/apps/chrome/extension/panel.css +++ b/apps/chrome/extension/panel.css @@ -1,2 +1,2 @@ :root{color-scheme:light dark;--bg:#f8faf9;--surface:#fff;--ink:#172a2a;--muted:#617675;--line:#dce5e2;--accent:#007f79;--soft:#e7f4ef;--error:#a43d24;font:14px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif} -*{box-sizing:border-box}body{margin:0;min-width:300px;height:100dvh;display:flex;flex-direction:column;background:var(--bg);color:var(--ink)}header{padding:18px 16px 10px;display:flex;align-items:center;justify-content:space-between;gap:6px}.brand{display:flex;align-items:center;gap:8px;letter-spacing:.02em}.mark{background:var(--accent);color:#fff;border-radius:8px;display:grid;place-items:center;width:28px;height:28px;font-size:19px;font-weight:750}.badge{color:var(--muted);font-size:9px;letter-spacing:.1em}button,input,textarea,select{font:inherit;color:inherit}button{cursor:pointer;background:var(--accent);color:#fff;border:0;border-radius:7px;padding:8px 12px;font-weight:600}button:disabled{opacity:.4;cursor:default}.quiet{background:transparent;color:var(--muted);font-size:12px;padding:6px}.quiet:hover:enabled{background:var(--soft);color:var(--accent)}:focus-visible{outline:2px solid var(--accent);outline-offset:2px}.connection{padding:0 16px 14px;display:flex;align-items:center;gap:7px;color:var(--muted);font-size:12px}.dot{height:7px;width:7px;border-radius:50%;background:var(--muted);flex-shrink:0}.dot.online{background:var(--accent)}.picker{padding:8px 16px 10px;border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.row{display:flex;align-items:center;justify-content:space-between;gap:8px}label{font-weight:650;font-size:12px}input,select,textarea{display:block;width:100%;border:1px solid var(--line);background:var(--surface);border-radius:7px;padding:10px}input{margin:5px 0 8px}select{font-weight:600;text-overflow:ellipsis}.hint{font-size:11px;color:var(--muted);margin:7px 0 0}.picker .hint{margin-top:6px}#notice{margin:10px 16px 0;padding:10px;background:var(--soft);border-left:3px solid var(--error);font-size:12px;overflow-wrap:anywhere}main{flex:1;min-height:100px;overflow-y:auto;padding:12px 16px;scrollbar-width:thin}.empty{padding:30px 6px;color:var(--muted)}.empty-mark{font-size:42px;color:var(--accent);font-weight:800}h1{font-size:27px;line-height:1.2;letter-spacing:-.04em;color:var(--ink);font-weight:600}.message{margin:8px 0 22px;overflow-wrap:anywhere}.message .role{font-weight:700;color:var(--muted);font-size:11px;margin-bottom:6px;letter-spacing:.05em}.message .text{white-space:pre-wrap;tab-size:2}.message.user .text{background:var(--soft);border-radius:9px;padding:10px 12px}.message.streaming .role{color:var(--accent)}.activity{margin:0;padding:3px 16px;color:var(--accent);font-size:12px;min-height:24px}form{padding:8px 16px 10px}textarea{resize:vertical;max-height:220px;min-height:76px}form>.row{margin-top:6px}#attachment{font-size:11px;display:flex;align-items:center;justify-content:space-between;gap:5px;background:var(--soft);border-radius:6px;padding:5px 8px;margin-bottom:6px}#attachment-label{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}footer{font-size:10px;display:flex;justify-content:space-between;padding:8px 16px 12px;color:var(--muted)}a{color:var(--muted);text-decoration:none}a:hover{text-decoration:underline}[hidden]{display:none!important}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0)}@media(prefers-color-scheme:dark){:root{--bg:#121c1c;--surface:#192525;--ink:#e1ede9;--muted:#94aaa5;--line:#2e4140;--accent:#43cabb;--soft:#203a34;--error:#e9997e}button{color:#10201d}}@media(max-width:340px){.badge{display:none}} +*{box-sizing:border-box}body{margin:0;min-width:300px;height:100dvh;display:flex;flex-direction:column;background:var(--bg);color:var(--ink)}header{padding:18px 16px 10px;display:flex;align-items:center;justify-content:space-between;gap:6px}.brand{display:flex;align-items:center;gap:8px;letter-spacing:.02em}.mark{background:var(--accent);color:#fff;border-radius:8px;display:grid;place-items:center;width:28px;height:28px;font-size:19px;font-weight:750}.badge{color:var(--muted);font-size:9px;letter-spacing:.1em}button,input,textarea,select{font:inherit;color:inherit}button{cursor:pointer;background:var(--accent);color:#fff;border:0;border-radius:7px;padding:8px 12px;font-weight:600}button:disabled{opacity:.4;cursor:default}.quiet{background:transparent;color:var(--muted);font-size:12px;padding:6px}.quiet:hover:enabled{background:var(--soft);color:var(--accent)}:focus-visible{outline:2px solid var(--accent);outline-offset:2px}.connection{padding:0 16px 14px;display:flex;align-items:center;gap:7px;color:var(--muted);font-size:12px}.dot{height:7px;width:7px;border-radius:50%;background:var(--muted);flex-shrink:0}.dot.online{background:var(--accent)}.picker{padding:8px 16px 10px;border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.row{display:flex;align-items:center;justify-content:space-between;gap:8px}label{font-weight:650;font-size:12px}input,select,textarea{display:block;width:100%;border:1px solid var(--line);background:var(--surface);border-radius:7px;padding:10px}input{margin:5px 0 8px}select{font-weight:600;text-overflow:ellipsis}.hint{font-size:11px;color:var(--muted);margin:7px 0 0}.picker .hint{margin-top:6px}#notice{margin:10px 16px 0;padding:10px;background:var(--soft);border-left:3px solid var(--error);font-size:12px;overflow-wrap:anywhere}main{flex:1;min-height:100px;overflow-y:auto;padding:12px 16px;scrollbar-width:thin}.empty{padding:30px 6px;color:var(--muted)}.empty-mark{font-size:42px;color:var(--accent);font-weight:800}h1{font-size:27px;line-height:1.2;letter-spacing:-.04em;color:var(--ink);font-weight:600}.message{margin:8px 0 22px;overflow-wrap:anywhere}.message .role{font-weight:700;color:var(--muted);font-size:11px;margin-bottom:6px;letter-spacing:.05em}.message .text{white-space:pre-wrap;tab-size:2}.message.user .text{background:var(--soft);border-radius:9px;padding:10px 12px}.message.streaming .role{color:var(--accent)}.activity{margin:0;padding:3px 16px;color:var(--accent);font-size:12px;min-height:24px}form{padding:8px 16px 10px}textarea{resize:vertical;max-height:220px;min-height:76px}form>.row{margin-top:6px}#attachment{font-size:11px;display:flex;align-items:center;justify-content:space-between;gap:5px;background:var(--soft);border-radius:6px;padding:5px 8px;margin-bottom:6px}#attachment-label{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}footer{font-size:10px;display:flex;justify-content:space-between;padding:8px 16px 12px;color:var(--muted)}a{color:var(--muted);text-decoration:none}a:hover{text-decoration:underline}[hidden]{display:none!important}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0)}.control{margin:0 16px 10px;padding:10px 12px;border:1px solid var(--line);border-radius:9px;background:var(--surface)}.control .hint{margin-top:6px}.control .actions{justify-content:flex-start;gap:4px;margin-top:4px}.toggle{display:flex;align-items:center;gap:8px;font-weight:650;font-size:12px;cursor:pointer;min-height:32px}.toggle input{width:18px;height:18px;margin:0;accent-color:var(--accent)}.stop{background:var(--error);color:#fff;padding:6px 12px}#control-details{margin-top:6px;font-size:11px;color:var(--muted)}#control-details summary{cursor:pointer;min-height:24px}#control-log{margin:6px 0 0;padding-left:18px;max-height:160px;overflow-y:auto}#control-log li{margin:3px 0;overflow-wrap:anywhere}#control-log li.failed{color:var(--error)}#control-log li.running{color:var(--accent)}@media(prefers-color-scheme:dark){:root{--bg:#121c1c;--surface:#192525;--ink:#e1ede9;--muted:#94aaa5;--line:#2e4140;--accent:#43cabb;--soft:#203a34;--error:#e9997e}button{color:#10201d}}@media(max-width:340px){.badge{display:none}} diff --git a/apps/chrome/extension/panel.html b/apps/chrome/extension/panel.html index e9ab42f..f465ecf 100644 --- a/apps/chrome/extension/panel.html +++ b/apps/chrome/extension/panel.html @@ -4,6 +4,12 @@
AAITURKCHROME BETA
IDE bağlantısı kuruluyor…
+
+
+

Durum okunuyor…

+
+ +
@@ -19,6 +25,6 @@

Sayfa yalnızca siz eklediğinizde okunur. Gönderilen mesajlar IDE’nin araçlarını çalıştırabilir. Onaylar IDE’de yanıtlanır.

- + diff --git a/apps/chrome/extension/panel.js b/apps/chrome/extension/panel.js index 8a4ffcb..7e173c5 100644 --- a/apps/chrome/extension/panel.js +++ b/apps/chrome/extension/panel.js @@ -112,9 +112,11 @@ function paintHistory(messages) { async function openSession(id) { const intent = ++generation + // A page sent from the context menu before any conversation was chosen + // waits for the first choice; switching conversations still clears it. + if (selected) { attachment = null; $('attachment').hidden = true } selected = id; opening = Boolean(id); stream = null bufferedEvents = [] - attachment = null; $('attachment').hidden = true $('message').value = ''; $('stop').hidden = true notice(''); controls() if (!id) { opening = false; $('transcript').replaceChildren(); activity('Bir konuşma seçin.'); return } @@ -203,20 +205,92 @@ $('stop').addEventListener('click', async () => { try { await request('message.stop', { sessionId: selected }); activity('Durdurma isteği IDE’ye iletildi.') } catch (error) { notice(error.message) } }) +function setAttachment(context) { + if (!context?.text) throw new Error('Bu sayfadan okunabilir metin alınamadı.') + attachment = context + $('attachment-label').textContent = `${context.selection ? 'Seçim' : 'Sayfa'}: ${context.title} · ${context.text.length.toLocaleString('tr')} karakter` + $('attachment').hidden = false + notice(selected ? 'Sayfa içeriği mesajınızla birlikte IDE’ye ve seçili model sağlayıcısına gönderilecek. Hassas bilgi içermediğinden emin olun.' + : 'Ek hazır. Göndermek için yukarıdan bir IDE konuşması seçin.') +} + +async function capture(tabId) { + const results = await chrome.scripting.executeScript({ target: { tabId }, func: capturePageContext }) + return results[0]?.result +} + $('add-page').addEventListener('click', async () => { const intent = generation try { const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) - if (!tab?.id || !/^https?:\/\//.test(tab.url || '')) throw new Error('Yalnızca normal HTTP/HTTPS sayfaları eklenebilir. Önce AITURK araç çubuğu simgesine tıklayarak bu sekmeye erişim verin.') - const results = await chrome.scripting.executeScript({ target: { tabId: tab.id }, func: capturePageContext }) + if (!tab?.id || !/^https?:\/\//.test(tab.url || '')) throw new Error('Yalnızca normal HTTP/HTTPS sayfaları eklenebilir.') + const context = await capture(tab.id) if (intent !== generation) return - const context = results[0]?.result - if (!context?.text) throw new Error('Bu sayfadan okunabilir metin alınamadı.') - attachment = context - $('attachment-label').textContent = `${context.selection ? 'Seçim' : 'Sayfa'}: ${context.title} · ${context.text.length.toLocaleString('tr')} karakter` - $('attachment').hidden = false - notice('Sayfa içeriği mesajınızla birlikte IDE’ye ve seçili model sağlayıcısına gönderilecek. Hassas bilgi içermediğinden emin olun.') + setAttachment(context) } catch (error) { notice(error.message) } }) + +// Text or a page sent from the right-click menu while the panel opens. +async function takePendingContext() { + const saved = await chrome.storage.session.get(['pendingContext', 'pendingNotice']) + if (saved.pendingNotice) { notice(saved.pendingNotice); await chrome.storage.session.remove('pendingNotice') } + const pending = saved.pendingContext + if (!pending) return + await chrome.storage.session.remove('pendingContext') + if (Date.now() - pending.at > 120000) return + try { + if (pending.kind === 'selection' && pending.text) { + const tab = await chrome.tabs.get(pending.tabId).catch(() => null) + let address = '' + try { const url = new URL(tab?.url || ''); address = url.origin + url.pathname } catch {} + setAttachment({ title: (tab?.title || '').slice(0, 200), url: address, text: pending.text.slice(0, 24000), selection: true }) + } else setAttachment(await capture(pending.tabId)) + } catch (error) { notice(error.message) } +} + +// ── The agent using this Chrome ─────────────────────────────────────── +const STATUS_MARK = { ok: '✓', failed: '✕', cancelled: '■', running: '…' } + +function renderControl(state) { + if (!state || typeof state !== 'object') return + $('control-enabled').checked = Boolean(state.enabled) + $('control-stop').hidden = !(state.enabled && state.running) + $('control-release').hidden = !(state.tabs?.length || state.shared) + let text + if (!state.enabled) text = 'Kapalı. Açarsan AITURK ajanı tarayıcı işlerini bu Chrome’da, kendi “AITURK” sekmelerinde yapar.' + else if (!state.registered) text = `IDE bekleniyor… AITURK IDE açık olmalı.${state.error ? ' (' + state.error + ')' : ''}` + else if (state.running) text = 'Ajan şu an bu Chrome’da çalışıyor. Durdurmak için “Durdur”a bas.' + else text = 'Hazır: ajan tarayıcı işlerini bu Chrome’da, “AITURK” sekme grubunda yapar. Şifre ve kart alanlarına yazmaz.' + if (state.shared) text += ` Paylaşılan sekme: “${state.shared.title || 'sekme'}”; ajan bir sonraki tarayıcı işinde bunu kullanır.` + if (state.tabs?.length) text += ` Ajanın sekmeleri: ${state.tabs.map(tab => tab.url || tab.title).filter(Boolean).join(', ')}.` + $('control-status').textContent = text + const items = (state.log || []).map(entry => { + const item = document.createElement('li') + item.className = entry.status + item.textContent = `${STATUS_MARK[entry.status] || ''} ${entry.text}${entry.note ? ' — ' + entry.note : ''}` + return item + }) + $('control-log').replaceChildren(...items) + $('control-details').hidden = !items.length +} + +async function control(type, extra = {}) { + try { + const reply = await chrome.runtime.sendMessage({ type, ...extra }) + if (reply?.failure) notice(reply.failure) + else renderControl(reply) + } catch (error) { notice(`Eklenti arka planına ulaşılamadı: ${error.message}`) } +} + +chrome.runtime.onMessage.addListener(message => { if (message?.type === 'control.state') renderControl(message.state) }) +chrome.storage.onChanged.addListener((changes, area) => { + if (area === 'session' && (changes.pendingContext?.newValue || changes.pendingNotice?.newValue)) void takePendingContext() +}) +$('control-enabled').addEventListener('change', () => void control('control.set', { enabled: $('control-enabled').checked })) +$('control-stop').addEventListener('click', () => void control('control.stop')) +$('control-share').addEventListener('click', () => void control('control.share', { sessionId: selected || null })) +$('control-release').addEventListener('click', () => void control('control.release')) $('remove-attachment').addEventListener('click', () => { attachment = null; $('attachment').hidden = true; notice('') }) void connect() +void control('control.get') +void takePendingContext() diff --git a/apps/chrome/tests/agent.test.mjs b/apps/chrome/tests/agent.test.mjs new file mode 100644 index 0000000..0008244 --- /dev/null +++ b/apps/chrome/tests/agent.test.mjs @@ -0,0 +1,339 @@ +import assert from 'node:assert/strict' +import { createRequire } from 'node:module' +import { test } from 'node:test' +import * as page from '../extension/page-agent.js' +import { CAPABILITIES, checkUrl, createController, parseKey, ToolError } from '../extension/controller.js' + +const { JSDOM } = createRequire(import.meta.url)('jsdom') +const GLOBALS = ['window', 'document', 'location', 'getComputedStyle', 'HTMLInputElement', 'HTMLTextAreaElement', 'Event', 'innerHeight', 'innerWidth'] + +function pageFixture(t, html) { + const dom = new JSDOM(`Sipariş formu${html}`, + { url: 'https://shop.example.test/checkout?session=gizli' }) + const { window } = dom + window.HTMLElement.prototype.getBoundingClientRect = function () { + if (this.dataset.size === '0') return { left: 0, top: 0, right: 0, bottom: 0, width: 0, height: 0 } + return { left: 10, top: 20, right: 110, bottom: 40, width: 100, height: 20 } + } + window.HTMLElement.prototype.scrollIntoView = () => {} + window.document.elementFromPoint = () => null + window.scrollBy = () => {} + for (const key of GLOBALS) globalThis[key] = key === 'innerHeight' ? 800 : key === 'innerWidth' ? 1200 : window[key] + t.after(() => { delete globalThis.__aiturkRefs; window.close(); for (const key of GLOBALS) delete globalThis[key] }) + return window +} + +const FORM = ` +

Ödeme

+ + Belgeler + + + + + + + +

Toplam tutar: 120 TL

+ + Ürün` + +test('snapshot lists visible controls with refs, masks passwords and skips hidden parts', t => { + pageFixture(t, FORM) + const result = page.snapshotPage({ full: false }) + const lines = result.snapshot.split('\n') + assert.equal(lines[0], 'Sayfa: Sipariş formu') + assert.ok(lines.includes('- heading "Ödeme" [level=1]')) + assert.ok(lines.includes('- link "Belgeler" [ref=e1]')) + assert.ok(lines.includes('- textbox "E-posta" [ref=e2] (değer: "ali@example.test")')) + const password = lines.find(line => line.includes('"Şifre"')) + assert.ok(password.includes('(değer: "••••")') && password.includes('şifre alanı: ajan yazamaz')) + assert.ok(!result.snapshot.includes('cok-gizli')) + assert.ok(!result.snapshot.includes('Gizli menü')) + assert.ok(!result.snapshot.includes('Görünmez') && !result.snapshot.includes('Sıfır boyut')) + assert.ok(lines.some(line => line.startsWith('- combobox "Şehir"') && line.includes('(değer: "İzmir")'))) + assert.ok(lines.some(line => line.startsWith('- checkbox "Koşulları kabul ediyorum"') && line.includes('[checked]'))) + assert.ok(!result.snapshot.includes('Toplam tutar'), 'compact view leaves plain text out') + assert.equal(result.element_count, 7) + assert.equal(globalThis.__aiturkRefs.size, 7) + + const full = page.snapshotPage({ full: true }) + assert.ok(full.snapshot.includes('- text: Toplam tutar: 120 TL')) + assert.ok(full.snapshot.includes('→ https://shop.example.test/docs')) + assert.ok(full.snapshot.includes('- option "Ankara"')) + + const short = page.snapshotPage({ limit: 2000, full: true }) + assert.ok(short.snapshot.length <= 2300) +}) + +test('a long page is cut with an explicit note instead of silently', t => { + pageFixture(t, Array.from({ length: 400 }, (_, index) => ``).join('')) + const result = page.snapshotPage({ limit: 2000 }) + assert.equal(result.truncated, true) + assert.match(result.snapshot, /satır daha var; kısaltıldı/) +}) + +test('refs resolve to coordinates; stale refs fail with a recoverable message', t => { + const window = pageFixture(t, FORM) + page.snapshotPage({}) + const located = page.locateRef('@e2') + assert.deepEqual([located.x, located.y, located.tag, located.password], [60, 30, 'input', false]) + window.document.getElementById('mail').remove() + assert.match(page.locateRef('@e2').error, /browser_snapshot/) + assert.match(page.locateRef('e99').error, /bulunamadı/) +}) + +test('typing refuses password and card fields, focuses text fields and fires framework events', t => { + const window = pageFixture(t, FORM) + page.snapshotPage({}) + const refs = [...globalThis.__aiturkRefs.keys()] + const refOf = predicate => `@${refs.find(ref => predicate(globalThis.__aiturkRefs.get(ref).deref()))}` + const passwordRef = refOf(element => element.type === 'password') + const cardRef = refOf(element => element.getAttribute('autocomplete') === 'cc-number') + assert.equal(page.prepareTyping(passwordRef).handoff, true) + assert.match(page.prepareTyping(cardRef).error, /Kart/) + assert.match(page.prepareTyping('@e1').error, /yazı alanı değil/) + + const mail = window.document.getElementById('mail') + let inputs = 0 + mail.addEventListener('input', () => inputs++) + assert.equal(page.prepareTyping('@e2').focused, true) + assert.deepEqual(page.typeWithEvents('@e2', 'veli@example.test'), { value: 'veli@example.test' }) + assert.equal(inputs, 1) + assert.deepEqual(page.readValue('@e2'), { value: 'veli@example.test' }) + + const selectRef = refOf(element => element.tagName === 'SELECT') + assert.deepEqual(page.prepareTyping(selectRef), { select: true }) + assert.deepEqual(page.typeWithEvents(selectRef, 'ankara'), { value: 'Ankara' }) + assert.match(page.typeWithEvents(selectRef, 'Paris').error, /seçeneği yok/) +}) + +test('images are listed with data URLs shortened; annotation labels come and go', t => { + const window = pageFixture(t, FORM) + const images = page.listImages() + assert.equal(images.count, 1) + assert.ok(images.images[0].src.length < 70 && images.images[0].alt === 'Ürün') + page.snapshotPage({}) + const labels = page.annotateRefs(true) + assert.equal(labels.annotations.length, 7) + assert.ok(window.document.getElementById('__aiturk_labels')) + page.annotateRefs(false) + assert.equal(window.document.getElementById('__aiturk_labels'), null) +}) + +test('keys and addresses are validated before anything touches the page', () => { + assert.deepEqual(parseKey('Enter'), { key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, text: '\r', modifiers: 0 }) + assert.deepEqual(parseKey('ctrl+a'), { key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65, text: undefined, modifiers: 2 }) + assert.equal(parseKey('Shift+Tab').modifiers, 8) + assert.equal(parseKey('esc').key, 'Escape') + assert.throws(() => parseKey('Hyper+x'), ToolError) + assert.throws(() => parseKey('NotAKey'), ToolError) + assert.equal(checkUrl('example.test/a?b=1'), 'https://example.test/a?b=1') + assert.equal(checkUrl('http://localhost:5173/'), 'http://localhost:5173/') + for (const blocked of ['chrome://settings', 'file:///C:/Windows/win.ini', 'javascript:alert(1)', 'chrome-extension://abc/panel.html', + 'https://chromewebstore.google.com/detail/x', 'https://user:pass@example.test/']) { + assert.throws(() => checkUrl(blocked), ToolError, blocked) + } +}) + +// ── controller with a fake Chrome ────────────────────────────────────── +const hub = () => ({ listeners: new Set(), addListener(fn) { this.listeners.add(fn) }, removeListener(fn) { this.listeners.delete(fn) }, + emit(...args) { for (const fn of [...this.listeners]) fn(...args) } }) +const pick = (store, keys) => Object.fromEntries([keys].flat().filter(key => key in store).map(key => [key, store[key]])) +const until = async (condition, label) => { + for (let index = 0; index < 400; index++) { if (condition()) return; await new Promise(resolve => setTimeout(resolve, 5)) } + assert.fail(`zaman aşımı: ${label}`) +} + +function fakeChrome({ pageScript = {}, tabs: initialTabs = [] } = {}) { + const local = {}, session = {}, native = [], debuggerCalls = [], notifications = [], groups = new Map() + const tabs = new Map(initialTabs.map(tab => [tab.id, { status: 'complete', windowId: 1, active: true, ...tab }])) + let nextTab = 100, nextGroup = 1, port = null + const onUpdated = hub() + const respond = message => { + const result = message.action === 'status' ? { profile: 'default', local: true } + : message.action === 'browser.register' ? { registered: message.params.enabled } + : message.action === 'sessions.list' ? { sessions: [{ id: 'durable-1', title: 'Rapor işi' }] } + : { accepted: true } + port.onMessage.emit({ id: message.id, result }) + } + const chrome = { + runtime: { + connectNative: () => { + port = { onMessage: hub(), onDisconnect: hub(), disconnected: false, disconnect() { this.disconnected = true }, + postMessage: message => { native.push(message); queueMicrotask(() => respond(message)) } } + return port + }, + sendMessage: async () => {}, + }, + storage: { + local: { get: async keys => pick(local, keys), set: async value => Object.assign(local, value) }, + session: { get: async keys => structuredClone(pick(session, keys)), set: async value => Object.assign(session, structuredClone(value)) }, + }, + windows: { getLastFocused: async () => ({ id: 1 }) }, + tabs: { + onUpdated, + create: async ({ url, active, windowId }) => { + const tab = { id: nextTab++, url, active, windowId, status: 'complete', title: '' } + tabs.set(tab.id, tab) + return { ...tab } + }, + get: async id => { if (!tabs.has(id)) throw new Error(`No tab with id: ${id}`); return { ...tabs.get(id) } }, + update: async (id, { url }) => { + const tab = tabs.get(id) + Object.assign(tab, { pendingUrl: url, status: 'loading' }) + setTimeout(() => { Object.assign(tab, { url, pendingUrl: undefined, status: 'complete', title: 'Açılan sayfa' }); onUpdated.emit(id, { status: 'complete' }) }, 20) + return { ...tab } + }, + goBack: async () => {}, + query: async () => [...tabs.values()].filter(tab => tab.active).map(tab => ({ ...tab })), + group: async ({ tabIds, groupId }) => { const id = groupId ?? nextGroup++; groups.set(id, { windowId: tabs.get(tabIds[0]).windowId }); return id }, + captureVisibleTab: async () => 'data:image/jpeg;base64,QUJD', + }, + tabGroups: { get: async id => ({ id, ...groups.get(id) }), update: async (id, value) => Object.assign(groups.get(id), value) }, + scripting: { + executeScript: async ({ target, func, args }) => { + const handler = pageScript[func.name] + if (!handler) throw new Error(`script not expected: ${func.name}`) + return [{ result: await handler(...args, target.tabId) }] + }, + }, + debugger: { + attach: async () => {}, detach: async () => {}, + sendCommand: async (target, method, params) => { debuggerCalls.push([method, params]); return method === 'Page.captureScreenshot' ? { data: 'QUJD' } : {} }, + }, + notifications: { create: async (...args) => { notifications.push(args) } }, + alarms: { create() {}, clear() {} }, + action: { setBadgeText: async () => {} }, + } + return { chrome, native, debuggerCalls, notifications, tabs, groups, local, session, get port() { return port } } +} + +const SNAPSHOT = { snapshot: 'Sayfa: Açılan sayfa\n- button "Gönder" [ref=e1]', element_count: 1, url: 'https://example.test/', title: 'Açılan sayfa' } + +async function connected(fake) { + const controller = createController({ chrome: fake.chrome, random: () => 'instance-1' }) + await controller.connect() + await until(() => controller._state().registered, 'registration') + return controller +} + +function command(fake, action, args, commandId = 'ab'.repeat(16)) { + fake.port.onMessage.emit({ event: 'browser.command', commandId, action, arguments: args, sessionId: 'durable-1' }) +} +const resultFor = (fake, commandId = 'ab'.repeat(16)) => fake.native.find(message => message.action === 'browser.result' && message.params.commandId === commandId) + +test('controller registers this Chrome for the IDE agent with a stable identity', async () => { + const fake = fakeChrome() + await connected(fake) + const register = fake.native.find(message => message.action === 'browser.register') + assert.deepEqual(register.params, { enabled: true, capabilities: [...CAPABILITIES], controllerId: 'aiturk-chrome-instance-1', browserProfileId: 'instance-1' }) + assert.equal(fake.local.controllerInstance, 'instance-1') +}) + +test('navigate opens the agent its own background tab in an AITURK group and returns a snapshot', async () => { + const fake = fakeChrome({ pageScript: { snapshotPage: () => SNAPSHOT } }) + await connected(fake) + command(fake, 'browser_navigate', { url: 'example.test' }) + await until(() => resultFor(fake), 'navigate result') + const { params } = resultFor(fake) + assert.equal(params.ok, true) + assert.equal(params.result.success, true) + assert.equal(params.result.snapshot, SNAPSHOT.snapshot) + const tab = fake.tabs.get(100) + assert.equal(tab.active, false) + assert.equal(tab.url, 'https://example.test/') + assert.equal(fake.groups.get(1).title, 'AITURK · Rapor işi') + assert.equal(fake.groups.get(1).color, 'cyan') +}) + +test('blocked pages and missing tabs come back as tool errors the agent can read', async () => { + const fake = fakeChrome() + await connected(fake) + command(fake, 'browser_navigate', { url: 'chrome://settings/passwords' }, '01'.repeat(16)) + command(fake, 'browser_click', { ref: '@e1' }, '02'.repeat(16)) + await until(() => resultFor(fake, '01'.repeat(16)) && resultFor(fake, '02'.repeat(16)), 'refusals') + assert.deepEqual(resultFor(fake, '01'.repeat(16)).params.result.success, false) + assert.match(resultFor(fake, '01'.repeat(16)).params.result.error, /çalışamaz/) + assert.match(resultFor(fake, '02'.repeat(16)).params.result.error, /browser_navigate/) + assert.equal(fake.tabs.size, 0, 'a refused navigation never opens a tab') +}) + +test('click is a real mouse click at the element; a covering element is reported', async () => { + const fake = fakeChrome({ tabs: [{ id: 7, url: 'https://example.test/', title: 'Örnek' }], + pageScript: { locateRef: () => ({ x: 60, y: 30, name: 'Gönder', covered: 'Çerez bildirimi' }) } }) + const controller = await connected(fake) + await controller.shareTab(await fake.chrome.tabs.get(7), 'durable-1') + command(fake, 'browser_click', { ref: '@e1' }) + await until(() => resultFor(fake), 'click result') + const { result } = resultFor(fake).params + assert.equal(result.success, true) + assert.match(result.warning, /Çerez bildirimi/) + const mouse = fake.debuggerCalls.filter(([method]) => method === 'Input.dispatchMouseEvent').map(([, params]) => [params.type, params.x, params.y]) + assert.deepEqual(mouse, [['mousePressed', 60, 30], ['mouseReleased', 60, 30]]) +}) + +test('password fields are handed to the user: nothing is typed and a notification is shown', async () => { + const fake = fakeChrome({ tabs: [{ id: 7, url: 'https://example.test/login' }], + pageScript: { prepareTyping: () => ({ error: 'Şifre alanına ajan yazamaz. Bu adımı kullanıcıya bırakın.', handoff: true }) } }) + const controller = await connected(fake) + await controller.shareTab(await fake.chrome.tabs.get(7), 'durable-1') + command(fake, 'browser_type', { ref: '@e2', text: 'parola' }) + await until(() => resultFor(fake), 'type result') + assert.equal(resultFor(fake).params.result.success, false) + assert.ok(!fake.debuggerCalls.some(([method]) => method === 'Input.insertText')) + assert.equal(fake.notifications.length, 1) +}) + +test('typing inserts trusted text and reports what the field now holds', async () => { + const fake = fakeChrome({ tabs: [{ id: 7, url: 'https://example.test/' }], + pageScript: { prepareTyping: () => ({ focused: true }), readValue: () => ({ value: 'İstanbul' }) } }) + const controller = await connected(fake) + await controller.shareTab(await fake.chrome.tabs.get(7), 'durable-1') + command(fake, 'browser_type', { ref: '@e3', text: 'İstanbul' }) + await until(() => resultFor(fake), 'type result') + assert.deepEqual(resultFor(fake).params.result, { success: true, typed: '@e3', value: 'İstanbul' }) + assert.deepEqual(fake.debuggerCalls.find(([method]) => method === 'Input.insertText')[1], { text: 'İstanbul' }) +}) + +test('vision returns a JPEG capture of the agent tab', async () => { + const fake = fakeChrome({ tabs: [{ id: 7, url: 'https://example.test/', title: 'Örnek' }] }) + const controller = await connected(fake) + await controller.shareTab(await fake.chrome.tabs.get(7), 'durable-1') + command(fake, 'browser_vision', { question: 'Ne var?' }) + await until(() => resultFor(fake), 'vision result') + assert.deepEqual(resultFor(fake).params.result, { success: true, screenshot: 'QUJD', mime: 'image/jpeg', url: 'https://example.test/', title: 'Örnek' }) +}) + +test('a cancelled command never answers late; stopping turns control off at the IDE', async () => { + let release + const fake = fakeChrome({ tabs: [{ id: 7, url: 'https://example.test/' }], + pageScript: { snapshotPage: () => new Promise(resolve => { release = () => resolve(SNAPSHOT) }) } }) + const controller = await connected(fake) + await controller.shareTab(await fake.chrome.tabs.get(7), 'durable-1') + command(fake, 'browser_snapshot', {}) + await until(() => release, 'snapshot started') + fake.port.onMessage.emit({ event: 'browser.cancel', commandId: 'ab'.repeat(16) }) + release() + await new Promise(resolve => setTimeout(resolve, 50)) + assert.equal(resultFor(fake), undefined) + + const port = fake.port + const state = await controller.onMessage({ type: 'control.stop' }) + assert.equal(state.enabled, false) + assert.deepEqual(fake.native.filter(message => message.action === 'browser.register').at(-1).params, { enabled: false }) + assert.equal(port.disconnected, true) + assert.equal(fake.local.controlEnabled, false) +}) + +test('cancelling Chrome’s debugging bar is treated as the user stopping the agent', async () => { + const fake = fakeChrome({ tabs: [{ id: 7, url: 'https://example.test/' }], + pageScript: { locateRef: () => ({ x: 1, y: 1, name: 'x' }) } }) + const controller = await connected(fake) + await controller.shareTab(await fake.chrome.tabs.get(7), 'durable-1') + command(fake, 'browser_click', { ref: '@e1' }) + await until(() => resultFor(fake), 'click result') + controller.onDebuggerDetach({ tabId: 7 }, 'canceled_by_user') + await until(() => fake.local.controlEnabled === false, 'disabled') + assert.equal(fake.notifications.length, 1) +}) diff --git a/apps/chrome/tests/bridge.test.mjs b/apps/chrome/tests/bridge.test.mjs index 3c2d161..557f174 100644 --- a/apps/chrome/tests/bridge.test.mjs +++ b/apps/chrome/tests/bridge.test.mjs @@ -151,6 +151,55 @@ test('real HTTP+WebSocket: same stored/runtime session, queue sends and preserve assert.ok(!JSON.stringify(events).includes('secret never')) }) +test('browser control: only a registered connection receives agent commands; results and refusals go back', async t => { + const f = await fixture(t), events = [] + const client = new ChromeGatewayClient(f.resolve, event => events.push(event)) + t.after(() => client.close()) + const call = (action, params = {}) => client.handle({ id: 'test', action, params }) + const command = (action, args = {}) => client.onEvent({ type: 'browser.controller.command', session_id: 'durable-1', + payload: { command_id: 'ab'.repeat(16), action, arguments: args, tool_call_id: 'call-1' } }) + + assert.throws(() => validateRequest({ id: 'a', action: 'browser.register', params: { enabled: true, capabilities: ['browser_cdp'], controllerId: 'c', browserProfileId: 'p' } })) + assert.throws(() => validateRequest({ id: 'a', action: 'browser.result', params: { commandId: '../x', ok: true } })) + assert.throws(() => validateRequest({ id: 'a', action: 'browser.result', params: { commandId: 'ab'.repeat(16), ok: false } })) + + await call('status') + command('browser_navigate', { url: 'https://example.test/' }) + assert.equal(events.length, 0, 'commands are never forwarded before the extension registers') + + await call('browser.register', { enabled: true, capabilities: ['browser_navigate', 'browser_vision'], controllerId: 'chrome-ext', browserProfileId: 'profile-1' }) + assert.deepEqual(f.calls.at(-1).params, { protocol_version: 1, controller_id: 'chrome-ext', browser_profile_id: 'profile-1', + capabilities: ['browser_navigate', 'browser_vision'], profile: 'default' }) + assert.equal(f.calls.at(-1).method, 'browser.controller.local.register') + + command('browser_navigate', { url: 'https://example.test/' }) + assert.deepEqual(events.at(-1), { event: 'browser.command', commandId: 'ab'.repeat(16), action: 'browser_navigate', + arguments: { url: 'https://example.test/' }, sessionId: 'durable-1' }) + + const before = f.calls.length + command('browser_evaluate', { expression: 'document.cookie' }) + command('browser_type', { ref: '@e1', text: 'x'.repeat(300 * 1024) }) + assert.equal(events.at(-1).action, 'browser_navigate', 'refused commands never reach the extension') + await new Promise(resolve => setTimeout(resolve, 50)) + const refusals = f.calls.slice(before) + assert.equal(refusals.length, 2) + assert.ok(refusals.every(frame => frame.method === 'browser.controller.local.result' && frame.params.ok === false)) + + client.onEvent({ type: 'browser.controller.cancel', payload: { command_id: 'cd'.repeat(16) } }) + assert.deepEqual(events.at(-1), { event: 'browser.cancel', commandId: 'cd'.repeat(16) }) + + await call('browser.result', { commandId: 'ab'.repeat(16), ok: true, result: { success: true, url: 'https://example.test/' } }) + assert.deepEqual(f.calls.at(-1).params, { command_id: 'ab'.repeat(16), ok: true, result: { success: true, url: 'https://example.test/' } }) + await call('browser.result', { commandId: 'ab'.repeat(16), ok: false, error: 'Sekme kapandı' }) + assert.deepEqual(f.calls.at(-1).params, { command_id: 'ab'.repeat(16), ok: false, error: 'Sekme kapandı' }) + + await call('browser.register', { enabled: false }) + assert.equal(f.calls.at(-1).method, 'browser.controller.local.detach') + const count = events.length + command('browser_navigate', { url: 'https://example.test/' }) + assert.equal(events.length, count, 'switching control off stops forwarding') +}) + test('remote and non-loopback descriptors are rejected before opening a socket', async () => { const client = new ChromeGatewayClient(async () => ({ baseUrl: 'https://example.test', wsUrl: 'wss://example.test/api/ws', token: 'x', mode: 'remote' }), () => {}) await assert.rejects(client.handle({ id: '1', action: 'status' }), /yalnızca/) diff --git a/apps/chrome/tests/panel.test.mjs b/apps/chrome/tests/panel.test.mjs index 6eb6ca1..97e5e6a 100644 --- a/apps/chrome/tests/panel.test.mjs +++ b/apps/chrome/tests/panel.test.mjs @@ -12,21 +12,33 @@ async function settle(condition) { assert.ok(condition(), 'UI did not settle') } -async function panel(t, respond) { +const CONTROL_READY = { enabled: true, connected: true, registered: true, running: 0, tabs: [], shared: null, + log: [{ status: 'ok', text: 'Sayfa açılıyor: example.test' }, { status: 'failed', text: 'Yazılıyor', note: 'Şifre alanına ajan yazamaz.' }] } + +async function panel(t, respond, { controlState = CONTROL_READY, session = {} } = {}) { const html = await readFile(new URL('../extension/panel.html', import.meta.url), 'utf8') const dom = new JSDOM(html, { url: 'https://extension-fixture.test/panel.html' }) - const calls = [] + const calls = [], controlCalls = [] const port = { onMessage: eventHub(), onDisconnect: eventHub(), disconnect() {}, postMessage(message) { calls.push(message) queueMicrotask(() => respond(message, frame => port.onMessage.emit(frame), port)) } } + const runtimeMessages = eventHub() + const sessionStore = { ...session } Object.assign(globalThis, { window: dom.window, document: dom.window.document, Option: dom.window.Option, - chrome: { runtime: { connectNative: () => port }, storage: { local: { get: async () => ({}), set: async () => {} } } } }) + chrome: { + runtime: { connectNative: () => port, onMessage: runtimeMessages, + sendMessage: async message => { controlCalls.push(message); return message.type === 'control.set' ? { ...controlState, enabled: message.enabled } : controlState } }, + storage: { local: { get: async () => ({}), set: async () => {} }, onChanged: eventHub(), + session: { get: async keys => Object.fromEntries([keys].flat().filter(key => key in sessionStore).map(key => [key, sessionStore[key]])), + remove: async key => { delete sessionStore[key] }, set: async value => Object.assign(sessionStore, value) } }, + tabs: { get: async id => ({ id, title: 'Kaynak sayfa', url: 'https://example.test/yazi?token=gizli#x' }) }, + } }) await import(`../extension/panel.js?test=${Math.random()}`) t.after(() => { dom.window.close(); for (const key of ['window', 'document', 'Option', 'chrome']) delete globalThis[key] }) const get = id => dom.window.document.getElementById(id) await settle(() => !get('sessions').disabled) - return { dom, calls, port, get, async select(id) { + return { dom, calls, controlCalls, runtimeMessages, port, get, async select(id) { get('sessions').value = id get('sessions').dispatchEvent(new dom.window.Event('change')) await settle(() => !get('sessions').disabled) @@ -101,3 +113,44 @@ test('explicit page capture excludes form values and strips URL credentials in q assert.equal(context.url, 'https://example.test/page') assert.equal(context.selection, false) }) + +test('agent control section shows the live state, never renders log text as HTML, and drives the background', async t => { + const p = await panel(t, fixtureResponse) + await settle(() => p.get('control-status').textContent.startsWith('Hazır')) + assert.equal(p.get('control-enabled').checked, true) + assert.equal(p.get('control-stop').hidden, true) + assert.equal(p.get('control-log').querySelector('b'), null) + assert.ok(p.get('control-log').textContent.includes('Yazılıyor')) + assert.ok(p.get('control-log').textContent.includes('Şifre alanına ajan yazamaz.')) + + p.runtimeMessages.emit({ type: 'control.state', state: { ...CONTROL_READY, running: 1 } }) + assert.equal(p.get('control-stop').hidden, false) + assert.ok(p.get('control-status').textContent.includes('çalışıyor')) + + await p.select('one') + p.get('control-share').dispatchEvent(new p.dom.window.Event('click')) + p.get('control-stop').dispatchEvent(new p.dom.window.Event('click')) + p.get('control-enabled').checked = false + p.get('control-enabled').dispatchEvent(new p.dom.window.Event('change')) + await settle(() => p.controlCalls.some(call => call.type === 'control.set')) + assert.deepEqual(p.controlCalls.find(call => call.type === 'control.share'), { type: 'control.share', sessionId: 'one' }) + assert.ok(p.controlCalls.some(call => call.type === 'control.stop')) + assert.deepEqual(p.controlCalls.find(call => call.type === 'control.set'), { type: 'control.set', enabled: false }) + await settle(() => p.get('control-status').textContent.startsWith('Kapalı')) +}) + +test('a selection sent from the right-click menu waits for the first conversation and drops URL secrets', async t => { + const p = await panel(t, fixtureResponse, { session: { pendingContext: { kind: 'selection', tabId: 7, text: 'Seçilen paragraf', at: Date.now() } } }) + await settle(() => !p.get('attachment').hidden) + assert.ok(p.get('attachment-label').textContent.includes('Seçim: Kaynak sayfa')) + assert.ok(p.get('notice').textContent.includes('konuşması seçin')) + await p.select('one') + assert.equal(p.get('attachment').hidden, false, 'the first choice keeps the pending attachment') + p.get('message').value = 'Bunu özetle' + p.get('composer').dispatchEvent(new p.dom.window.Event('submit', { cancelable: true })) + await settle(() => p.calls.some(call => call.action === 'message.send')) + const sent = p.calls.find(call => call.action === 'message.send').params.text + assert.ok(sent.includes('Seçilen paragraf')) + assert.ok(sent.includes('https://example.test/yazi')) + assert.ok(!sent.includes('token=gizli')) +}) diff --git a/apps/desktop/electron/chrome-bridge.mjs b/apps/desktop/electron/chrome-bridge.mjs index 6d5c917..fcb1131 100644 --- a/apps/desktop/electron/chrome-bridge.mjs +++ b/apps/desktop/electron/chrome-bridge.mjs @@ -31,12 +31,33 @@ export function publicMessages(rows) { return messages } +// Browser actions the extension may offer the IDE agent. The backend applies +// its own allowlist again; this one keeps unknown names off the wire. +export const BROWSER_CAPABILITIES = Object.freeze(['browser_navigate', 'browser_snapshot', 'browser_click', + 'browser_type', 'browser_scroll', 'browser_back', 'browser_press', 'browser_get_images', 'browser_vision']) +const COMMAND_ID = /^[a-f0-9]{16,64}$/ + export function validateRequest(message) { if (!message || typeof message.id !== 'string' || message.id.length > 80 || - !['status', 'sessions.list', 'session.open', 'session.history', 'message.send', 'message.stop'].includes(message.action)) { + !['status', 'sessions.list', 'session.open', 'session.history', 'message.send', 'message.stop', + 'browser.register', 'browser.result'].includes(message.action)) { throw new Error('Desteklenmeyen eklenti isteği.') } const params = message.params || {} + if (message.action === 'browser.register') { + if (typeof params.enabled !== 'boolean') throw new Error('Tarayıcı denetimi ayarı geçersiz.') + if (params.enabled && (!Array.isArray(params.capabilities) || !params.capabilities.length || + params.capabilities.some(name => !BROWSER_CAPABILITIES.includes(name)) || + !/^[a-zA-Z0-9_.:-]{1,120}$/.test(params.controllerId || '') || + !/^[a-zA-Z0-9_.:-]{1,120}$/.test(params.browserProfileId || ''))) { + throw new Error('Tarayıcı denetimi kaydı geçersiz.') + } + } + if (message.action === 'browser.result' && + (typeof params.commandId !== 'string' || !COMMAND_ID.test(params.commandId) || typeof params.ok !== 'boolean' || + (!params.ok && (typeof params.error !== 'string' || params.error.length > 4000)))) { + throw new Error('Tarayıcı komut sonucu geçersiz.') + } if (['session.open', 'session.history', 'message.send', 'message.stop'].includes(message.action) && (typeof params.sessionId !== 'string' || !/^[a-zA-Z0-9_.:-]{1,160}$/.test(params.sessionId))) { throw new Error('Geçersiz konuşma kimliği.') @@ -57,6 +78,8 @@ export class ChromeGatewayClient { this.nextId = 0 this.selected = null this.closed = false + // True once this connection offered the user's browser to the IDE agent. + this.controller = false } async connect() { @@ -92,6 +115,8 @@ export class ChromeGatewayClient { } this.pending.clear() this.selected = null + // The backend drops a local browser controller with its transport. + this.controller = false if (!this.closed) this.emit({ event: 'disconnected', message: 'IDE bağlantısı kesildi. Yeniden bağlanın.' }) }) await new Promise((resolve, reject) => { @@ -135,6 +160,10 @@ export class ChromeGatewayClient { } onEvent(event) { + if (this.controller && ['browser.controller.command', 'browser.controller.cancel'].includes(event?.type)) { + this.onControllerEvent(event) + return + } const selected = this.selected if (!selected || event?.session_id !== selected.runtimeId) return const payload = event.payload || {} @@ -152,6 +181,26 @@ export class ChromeGatewayClient { } } + onControllerEvent(event) { + const payload = event.payload || {} + const commandId = payload.command_id + if (typeof commandId !== 'string' || !COMMAND_ID.test(commandId)) return + if (event.type === 'browser.controller.cancel') { + this.emit({ event: 'browser.cancel', commandId }) + return + } + const refuse = error => void this.rpc('browser.controller.local.result', + { command_id: commandId, ok: false, error }).catch(() => {}) + if (!BROWSER_CAPABILITIES.includes(payload.action)) return refuse('Chrome bu tarayıcı işlemini desteklemiyor.') + const args = payload.arguments && typeof payload.arguments === 'object' && !Array.isArray(payload.arguments) + ? payload.arguments : {} + // A frame over the native limit would drop the whole connection; refuse + // the one oversized command instead. + if (Buffer.byteLength(JSON.stringify(args)) > 256 * 1024) return refuse('Tarayıcı komutu çok büyük.') + this.emit({ event: 'browser.command', commandId, action: payload.action, arguments: args, + sessionId: typeof event.session_id === 'string' ? event.session_id.slice(0, 160) : '' }) + } + async handle(message) { const params = validateRequest(message) await this.connect() @@ -193,6 +242,24 @@ export class ChromeGatewayClient { } case 'session.history': return this.history(params.sessionId) + case 'browser.register': { + if (!params.enabled) { + this.controller = false + const result = await this.rpc('browser.controller.local.detach', {}) + return { registered: false, detached: Boolean(result?.detached) } + } + const result = await this.rpc('browser.controller.local.register', { protocol_version: 1, + controller_id: params.controllerId, browser_profile_id: params.browserProfileId, + capabilities: params.capabilities, profile: this.backend.profile || 'default' }) + this.controller = true + return { registered: true, capabilities: Array.isArray(result?.capabilities) ? result.capabilities : [] } + } + case 'browser.result': { + const result = await this.rpc('browser.controller.local.result', params.ok + ? { command_id: params.commandId, ok: true, result: params.result ?? null } + : { command_id: params.commandId, ok: false, error: params.error }) + return { accepted: Boolean(result?.accepted) } + } case 'message.send': case 'message.stop': { if (this.selected?.storedId !== params.sessionId) throw new Error('Önce bu konuşmayı seçin.') diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 15a6a5f..a9a5224 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -2,7 +2,7 @@ "name": "aiturk-ide", "productName": "AITURK IDE", "private": true, - "version": "1.0.0-beta.41", + "version": "1.0.0-beta.42", "description": "TurkServis resmi yapay zekâ geliştirme ortamı. Hermes Agent tabanlıdır.", "author": "AITURK / TurkServis", "repository": { diff --git a/gateway/browser_control_broker.py b/gateway/browser_control_broker.py index d794f09..8af9e46 100644 --- a/gateway/browser_control_broker.py +++ b/gateway/browser_control_broker.py @@ -69,7 +69,7 @@ import logging import secrets import threading import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Callable, Dict, Optional logger = logging.getLogger(__name__) @@ -94,6 +94,7 @@ BROWSER_CONTROL_CAPABILITIES = frozenset( "controller.noop", "browser_back", "browser_click", + "browser_get_images", "browser_navigate", "browser_press", "browser_screenshot", @@ -102,9 +103,22 @@ BROWSER_CONTROL_CAPABILITIES = frozenset( "browser_tab_activate", "browser_tabs", "browser_type", + # The controller only captures the visible tab; the agent side saves + # the image and runs the usual native/auxiliary vision analysis. + "browser_vision", } ) +#: Transport family for the desktop IDE's own browser (the AITURK Chrome +#: extension behind the desktop's native-messaging bridge). Its controller is +#: registered once per backend as a lane template and serves every desktop +#: session of that backend; see :meth:`BrowserControlBroker.attach_lane_template`. +LOCAL_DESKTOP_TRANSPORT_FAMILY = "desktop-local-ws" +#: Principal of the single local desktop user. The desktop backend is +#: loopback-only and its session token already grants every desktop session, +#: so there is exactly one local principal per backend process. +LOCAL_DESKTOP_PRINCIPAL = "principal:desktop:local" + #: Privileged capabilities (Phase 8 Task 30) that are never negotiable through #: the base allowlist. ``browser_evaluate`` executes JavaScript in the page #: context; ``browser_cdp`` is raw CDP. Both are fail-closed unless the broker @@ -304,6 +318,23 @@ class _PendingCommand: result: Any = None +@dataclass +class _LaneTemplate: + """One controller offered to every session of one authenticated lane. + + A template never matches a session by itself. The first lookup for a + session materializes an exact per-session scope through :meth:`attach`, + so selection, dispatch, cancellation and completion keep the broker's + exact-identity contract. ``send_for(session_id)`` returns the transport + writer that addresses that session's frames. + """ + + template: ControllerScope + send_for: Callable[[str], Callable[[dict], None]] + owner: Any + derived: Dict[str, ControllerScope] = field(default_factory=dict) + + class BrowserControlBroker: """Thread-safe broker core binding controllers to callers. @@ -347,6 +378,10 @@ class BrowserControlBroker: # listener attaches one store per profile so profile A touching the # artifact route first can never pin profile B to A's physical root. self._artifact_stores: Dict[Optional[str], Any] = {} + # Lane templates keyed by (principal_id, transport_family). Only the + # local desktop lane uses them; every other transport registers one + # exact scope per session. + self._lane_templates: Dict[tuple, _LaneTemplate] = {} def _developer_mode_now(self) -> bool: """Current Developer Mode authority (live config unless pinned).""" @@ -575,6 +610,163 @@ class BrowserControlBroker: ] return len(matches) == 1 + # ------------------------------------------------------------------ + # Lane templates (one controller for every session of a lane) + # ------------------------------------------------------------------ + + def attach_lane_template( + self, + template: ControllerScope, + send_for: Callable[[str], Callable[[dict], None]], + *, + owner: Any, + ) -> None: + """Offer one controller to every session of ``template``'s lane. + + The lane is ``(principal_id, transport_family)``; ``template`` carries + no session id. Re-registering the same controller (same profile, + controller id and browser profile) refreshes the transport and + negotiated capabilities of the sessions it already serves without + cancelling their work. A different controller hard-replaces the old + one: its per-session scopes are detached first, cancelling their + pending commands, exactly like a replacement in :meth:`attach`. + """ + template = replace(template, session_id=None) + key = (template.principal_id, template.transport_family) + lane = _LaneTemplate(template=template, send_for=send_for, owner=owner) + with self._lock: + previous = self._lane_templates.get(key) + self._lane_templates[key] = lane + if previous is None: + return + same_controller = ( + previous.template.profile_id == template.profile_id + and previous.template.controller_id == template.controller_id + and previous.template.browser_profile_id == template.browser_profile_id + ) + for session_id, scope in list(previous.derived.items()): + if not same_controller: + self._detach_identity(scope, notify_controller=False) + continue + refreshed = replace(template, session_id=session_id) + try: + self.attach(refreshed, send_for(session_id), owner=owner) + except Exception: + logger.exception("failed to refresh browser-controller lane scope") + self._detach_identity(scope, notify_controller=False) + continue + with self._lock: + if self._lane_templates.get(key) is lane: + lane.derived[session_id] = refreshed + + def detach_lane_template( + self, + *, + principal_id: Optional[str], + transport_family: Optional[str], + owner: Any = _OWNER_UNSET, + notify_controller: bool = True, + ) -> bool: + """Remove a lane template and hard-detach every session it served.""" + key = (principal_id, transport_family) + with self._lock: + lane = self._lane_templates.get(key) + if lane is None: + return False + if owner is not _OWNER_UNSET and lane.owner is not owner: + return False + del self._lane_templates[key] + derived = list(lane.derived.values()) + for scope in derived: + self._detach_identity(scope, notify_controller=notify_controller) + return True + + def lane_template( + self, *, principal_id: Optional[str], transport_family: Optional[str] + ) -> Optional[ControllerScope]: + """Return the template scope offered to a lane, if any.""" + with self._lock: + lane = self._lane_templates.get((principal_id, transport_family)) + return lane.template if lane is not None else None + + def _materialize_lane_scope( + self, target: str, principal: str, family: str + ) -> Optional[ControllerScope]: + """Attach the exact per-session scope a lane template offers.""" + key = (principal, family) + with self._lock: + lane = self._lane_templates.get(key) + if lane is None: + return None + existing = lane.derived.get(target) + if existing is not None and any( + _same_scope_identity(candidate, existing) + for candidate in self._controllers + ): + return existing + scope = replace(lane.template, session_id=target) + send_for, owner = lane.send_for, lane.owner + self.attach(scope, send_for(target), owner=owner) + with self._lock: + if self._lane_templates.get(key) is lane: + lane.derived[target] = scope + return scope + # The template was replaced or removed while attaching: never leave a + # scope behind that its lane no longer vouches for. + self._detach_identity(scope, notify_controller=False) + return None + + def _detach_identity( + self, scope: ControllerScope, *, notify_controller: bool = True + ) -> None: + """Detach whatever scope currently carries ``scope``'s identity. + + A refresh may have re-keyed the scope with new capabilities, so an + exact-key lookup could miss it. + """ + with self._lock: + current = next( + ( + candidate + for candidate in self._controllers + if _same_scope_identity(candidate, scope) + ), + None, + ) + if current is not None: + self.detach(current, notify_controller=notify_controller) + + def complete_for_owner( + self, + command_id: str, + *, + owner: Any, + ok: bool, + result: Any = None, + ) -> bool: + """Complete a command only when ``owner`` drives its controller. + + Lane controllers serve many sessions over one transport, so their + results are admitted by transport ownership instead of a session + lookup. Unknown, foreign or already-resolved ids report ``False``. + """ + with self._lock: + pending = self._pending.get(command_id) + if pending is None or pending.done: + return False + controller = next( + ( + candidate + for candidate in self._controllers.values() + if _same_scope_identity(candidate.scope, pending.scope) + ), + None, + ) + if controller is None or controller.owner is not owner: + return False + scope = pending.scope + return self.complete(command_id, scope=scope, ok=ok, result=result) + def disconnect( self, scope: ControllerScope, @@ -951,7 +1143,12 @@ class BrowserControlBroker: and scope.principal_id == principal and scope.transport_family == family ] - return matches[0] if len(matches) == 1 else None + has_template = (principal, family) in self._lane_templates + if matches: + return matches[0] if len(matches) == 1 else None + if has_template: + return self._materialize_lane_scope(target, principal, family) + return None def lane_registered( self, @@ -975,6 +1172,8 @@ class BrowserControlBroker: if not target or not principal or not family: return False with self._lock: + if (principal, family) in self._lane_templates: + return True return any( scope.session_id == target and scope.principal_id == principal @@ -982,8 +1181,30 @@ class BrowserControlBroker: for scope in self._controllers ) + def _detach_lane_templates_owned_by(self, owner: Any) -> int: + with self._lock: + keys = [ + key + for key, lane in self._lane_templates.items() + if lane.owner is owner + ] + for principal_id, transport_family in keys: + self.detach_lane_template( + principal_id=principal_id, + transport_family=transport_family, + owner=owner, + ) + return len(keys) + def disconnect_owner(self, owner: Any) -> int: - """Mark every controller owned by one lost transport offline.""" + """Mark every controller owned by one lost transport offline. + + Lane templates are the exception: a lane controller that goes away + (the user closed Chrome, or the IDE bridge reconnected) is removed + outright so its sessions go back to their normal browser instead of + failing closed until some later reconnect. + """ + self._detach_lane_templates_owned_by(owner) with self._lock: scopes = [ scope @@ -997,6 +1218,7 @@ class BrowserControlBroker: def detach_owner(self, owner: Any, *, notify_controller: bool = True) -> int: """Hard-detach every controller owned by one transport connection.""" + self._detach_lane_templates_owned_by(owner) with self._lock: scopes = [ scope @@ -1014,6 +1236,7 @@ class BrowserControlBroker: def reset(self) -> None: """Fail all live work closed and clear tickets (tests/shutdown).""" with self._lock: + self._lane_templates.clear() scopes = list(self._controllers) for scope in scopes: self.detach(scope) @@ -1065,3 +1288,30 @@ def browser_control_enabled(config: Optional[dict] = None) -> bool: if not isinstance(extension_control, dict): return False return extension_control.get("enabled", False) is True + + +def browser_control_local_enabled(config: Optional[dict] = None) -> bool: + """Whether the desktop IDE's own browser may become a lane controller. + + Separate from :func:`browser_control_enabled` because the local lane is a + different trust boundary: the controller is the user's own browser on the + loopback desktop backend, so it is on unless the user switched it off + (``browser.extension_control.local_desktop: false``). Registering the + controller is still an explicit act of the browser extension. + """ + if config is None: + try: + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() + except Exception: + return True + if not isinstance(config, dict): + return True + browser = config.get("browser") + if not isinstance(browser, dict): + return True + extension_control = browser.get("extension_control") + if not isinstance(extension_control, dict): + return True + return extension_control.get("local_desktop", True) is not False diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 16416b5..2f0da5a 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -697,6 +697,10 @@ DEFAULT_CONFIG = { "extension_control": { "enabled": False, "developer_mode": False, + # The desktop IDE's own browser (AITURK Chrome behind the native + # bridge) may serve desktop sessions once the extension registers. + # Independent of ``enabled``, which gates remote/API controllers. + "local_desktop": True, }, }, diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 0324ead..286143b 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -16727,6 +16727,9 @@ def _ws_auth_reason(ws: "WebSocket") -> tuple[Optional[str], str]: if not token: return "no_credential", "none" if hmac.compare_digest(token.encode(), _SESSION_TOKEN.encode()): + # Only reachable on the loopback / --insecure path: the desktop IDE and + # the bridges it spawns. Marks the local desktop browser-controller lane. + ws._hermes_local_desktop = True return None, "token" return "token_mismatch", "token" @@ -17864,6 +17867,7 @@ async def gateway_ws(ws: WebSocket) -> None: ws, auth_identity=getattr(ws, "_hermes_auth_identity", None), subprotocol=getattr(ws, "_hermes_ws_subprotocol", None), + local_desktop=getattr(ws, "_hermes_local_desktop", False) is True, ) diff --git a/package-lock.json b/package-lock.json index a45ff97..eb9a99c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,7 +65,7 @@ }, "apps/desktop": { "name": "aiturk-ide", - "version": "1.0.0-beta.41", + "version": "1.0.0-beta.42", "license": "MIT", "dependencies": { "@assistant-ui/core": "0.2.23", diff --git a/tests/gateway/test_browser_control_api.py b/tests/gateway/test_browser_control_api.py index d197352..01b2ca8 100644 --- a/tests/gateway/test_browser_control_api.py +++ b/tests/gateway/test_browser_control_api.py @@ -24,6 +24,7 @@ CONTROL_PROTOCOL = "hermes-browser-control-v1" REAL_BROWSER_CAPABILITIES = { "browser_back", "browser_click", + "browser_get_images", "browser_navigate", "browser_press", "browser_screenshot", @@ -32,6 +33,7 @@ REAL_BROWSER_CAPABILITIES = { "browser_tab_activate", "browser_tabs", "browser_type", + "browser_vision", } diff --git a/tests/gateway/test_browser_control_local_lane.py b/tests/gateway/test_browser_control_local_lane.py new file mode 100644 index 0000000..cc5fb12 --- /dev/null +++ b/tests/gateway/test_browser_control_local_lane.py @@ -0,0 +1,345 @@ +"""Local desktop browser lane: the AITURK Chrome extension behind the IDE. + +One controller registered from the loopback desktop transport serves every +desktop session of the backend; each session still gets an exact scope. +""" + +import base64 +import json +import threading +from types import SimpleNamespace + +import pytest + +from gateway.browser_control_broker import ( + LOCAL_DESKTOP_PRINCIPAL, + LOCAL_DESKTOP_TRANSPORT_FAMILY, + BrowserControlBroker, + ControllerCancelled, + ControllerScope, + get_browser_control_broker, +) +from hermes_cli import web_server +from tools.browser_extension_router import route_browser_tool +from tui_gateway import server + +CAPS = frozenset({"browser_navigate", "browser_snapshot", "browser_vision"}) + + +def _template(controller_id="chrome-1", capabilities=CAPS): + return ControllerScope( + principal_id=LOCAL_DESKTOP_PRINCIPAL, + profile_id="default", + session_id=None, + controller_id=controller_id, + browser_profile_id="chrome-profile", + transport_family=LOCAL_DESKTOP_TRANSPORT_FAMILY, + capabilities=capabilities, + ) + + +class _Writer: + def __init__(self): + self.frames = [] + self.sent = threading.Event() + + def send_for(self, session_id): + def send(frame): + self.frames.append((session_id, frame)) + self.sent.set() + + return send + + +def _lookup(broker, session_id): + return broker.scope_for_session( + session_id=session_id, + principal_id=LOCAL_DESKTOP_PRINCIPAL, + transport_family=LOCAL_DESKTOP_TRANSPORT_FAMILY, + ) + + +def _dispatch_async(broker, scope, action="browser_navigate"): + outcome = {} + + def run(): + try: + outcome["result"] = broker.dispatch( + scope, action=action, arguments={"url": "https://example.com"}, tool_call_id="call-1" + ) + except Exception as exc: # noqa: BLE001 - asserted by the tests + outcome["error"] = exc + + thread = threading.Thread(target=run) + thread.start() + return thread, outcome + + +def test_lane_template_serves_every_session_with_its_own_exact_scope(): + broker = BrowserControlBroker(command_timeout=5) + writer = _Writer() + owner = object() + assert broker.lane_registered( + session_id="a", principal_id=LOCAL_DESKTOP_PRINCIPAL, transport_family=LOCAL_DESKTOP_TRANSPORT_FAMILY + ) is False + broker.attach_lane_template(_template(), writer.send_for, owner=owner) + + first, second = _lookup(broker, "session-a"), _lookup(broker, "session-b") + assert first.session_id == "session-a" and second.session_id == "session-b" + assert _lookup(broker, "session-a") == first + assert broker.select(first, "browser_navigate") is not None + assert broker.select(first, "browser_click") is None + assert broker.lane_registered( + session_id="anything", principal_id=LOCAL_DESKTOP_PRINCIPAL, transport_family=LOCAL_DESKTOP_TRANSPORT_FAMILY + ) + + thread, outcome = _dispatch_async(broker, second) + assert writer.sent.wait(2) + session_id, frame = writer.frames[-1] + assert session_id == "session-b" + command_id = frame["params"]["command_id"] + assert broker.complete_for_owner(command_id, owner=object(), ok=True, result="x") is False + assert broker.complete_for_owner(command_id, owner=owner, ok=True, result={"success": True}) is True + thread.join(2) + assert outcome == {"result": {"success": True}} + + +def test_other_lanes_never_see_the_local_template(): + broker = BrowserControlBroker() + broker.attach_lane_template(_template(), _Writer().send_for, owner=object()) + assert broker.scope_for_session( + session_id="session-a", principal_id="principal:dashboard:x", transport_family="cloud-ticket-ws" + ) is None + assert broker.scope_for_session( + session_id="session-a", principal_id=LOCAL_DESKTOP_PRINCIPAL, transport_family="cloud-ticket-ws" + ) is None + + +def test_losing_the_transport_removes_the_lane_and_cancels_its_work(): + broker = BrowserControlBroker(command_timeout=5) + writer, owner = _Writer(), object() + broker.attach_lane_template(_template(), writer.send_for, owner=owner) + scope = _lookup(broker, "session-a") + thread, outcome = _dispatch_async(broker, scope) + assert writer.sent.wait(2) + + broker.disconnect_owner(owner) + thread.join(2) + assert isinstance(outcome.get("error"), ControllerCancelled) + assert _lookup(broker, "session-a") is None + assert broker.lane_registered( + session_id="session-a", principal_id=LOCAL_DESKTOP_PRINCIPAL, transport_family=LOCAL_DESKTOP_TRANSPORT_FAMILY + ) is False + + +def test_same_controller_refresh_keeps_work_but_a_new_controller_replaces_it(): + broker = BrowserControlBroker(command_timeout=5) + writer, owner = _Writer(), object() + broker.attach_lane_template(_template(), writer.send_for, owner=owner) + scope = _lookup(broker, "session-a") + thread, outcome = _dispatch_async(broker, scope) + assert writer.sent.wait(2) + command_id = writer.frames[-1][1]["params"]["command_id"] + + refreshed_writer = _Writer() + broker.attach_lane_template( + _template(capabilities=CAPS | {"browser_click"}), refreshed_writer.send_for, owner=owner + ) + refreshed = _lookup(broker, "session-a") + assert broker.select(refreshed, "browser_click") is not None + assert broker.complete_for_owner(command_id, owner=owner, ok=True, result="done") is True + thread.join(2) + assert outcome == {"result": "done"} + + thread, outcome = _dispatch_async(broker, refreshed) + assert refreshed_writer.sent.wait(2) + broker.attach_lane_template(_template(controller_id="chrome-2"), _Writer().send_for, owner=owner) + thread.join(2) + assert isinstance(outcome.get("error"), ControllerCancelled) + assert _lookup(broker, "session-a").controller_id == "chrome-2" + + +def test_router_uses_local_lane_without_the_remote_feature_flag(monkeypatch): + monkeypatch.setattr("gateway.browser_control_broker.browser_control_enabled", lambda config=None: False) + monkeypatch.setattr("gateway.browser_control_broker.browser_control_local_enabled", lambda config=None: True) + broker = BrowserControlBroker(command_timeout=5) + calls = [] + + def fallback(): + calls.append("legacy") + return "legacy" + + identity = dict( + session_id="session-a", + principal_id=LOCAL_DESKTOP_PRINCIPAL, + transport_family=LOCAL_DESKTOP_TRANSPORT_FAMILY, + ) + # No controller registered yet: the normal browser keeps working. + assert route_browser_tool("browser_navigate", {}, fallback=fallback, broker=broker, enabled=True, **identity) == "legacy" + + owner = object() + + def send_for(session_id): + def send(frame): + broker.complete_for_owner( + frame["params"]["command_id"], owner=owner, ok=True, result={"success": True, "screenshot": "eA=="} + ) + + return send + + broker.attach_lane_template(_template(), send_for, owner=owner) + seen = [] + result = route_browser_tool( + "browser_vision", {"question": "q"}, fallback=fallback, broker=broker, enabled=True, + transform=lambda value: seen.append(value) or {"envelope": True}, **identity, + ) + assert result == {"envelope": True} + assert seen == [{"success": True, "screenshot": "eA=="}] + assert calls == ["legacy"] + + +def test_lane_switch_off_keeps_the_normal_browser(monkeypatch): + from tools import browser_extension_router as router + + monkeypatch.setattr("gateway.browser_control_broker.browser_control_local_enabled", lambda config=None: False) + assert router.lane_enabled(LOCAL_DESKTOP_TRANSPORT_FAMILY) is False + monkeypatch.setattr("gateway.browser_control_broker.browser_control_local_enabled", lambda config=None: True) + assert router.lane_enabled(LOCAL_DESKTOP_TRANSPORT_FAMILY) is True + + +def test_local_switch_defaults_on_and_respects_explicit_off(): + from gateway.browser_control_broker import browser_control_local_enabled + + assert browser_control_local_enabled({}) is True + assert browser_control_local_enabled({"browser": {"extension_control": {"enabled": False}}}) is True + assert browser_control_local_enabled({"browser": {"extension_control": {"local_desktop": False}}}) is False + + +def test_only_the_loopback_token_marks_a_local_desktop_transport(): + token_ws = SimpleNamespace( + query_params={"token": web_server._SESSION_TOKEN}, headers={}, + client=SimpleNamespace(host="127.0.0.1"), url=SimpleNamespace(path="/api/ws"), + ) + previous = getattr(web_server.app.state, "auth_required", False) + web_server.app.state.auth_required = False + try: + assert web_server._ws_auth_ok(token_ws) is True + assert token_ws._hermes_local_desktop is True + wrong = SimpleNamespace( + query_params={"token": "not-the-token"}, headers={}, + client=SimpleNamespace(host="127.0.0.1"), url=SimpleNamespace(path="/api/ws"), + ) + assert web_server._ws_auth_ok(wrong) is False + assert getattr(wrong, "_hermes_local_desktop", False) is False + finally: + web_server.app.state.auth_required = previous + + +def test_desktop_session_context_binds_the_local_lane_identity(): + from gateway.session_context import clear_session_vars, get_session_env + + server._sessions["local-context-fixture"] = { + "transport": SimpleNamespace(auth_identity=None, local_desktop=True), + "session_key": "stored-local-context", + "profile": None, + "agent": SimpleNamespace(session_id="durable-local-session"), + } + tokens = [] + try: + tokens = server._set_session_context("stored-local-context") + assert get_session_env("HERMES_SESSION_ID") == "durable-local-session" + assert get_session_env("HERMES_BROWSER_CONTROL_PRINCIPAL") == LOCAL_DESKTOP_PRINCIPAL + assert get_session_env("HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY") == LOCAL_DESKTOP_TRANSPORT_FAMILY + finally: + clear_session_vars(tokens) + server._sessions.pop("local-context-fixture", None) + + +def _rpc(method, params, transport): + return server.dispatch({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}, transport) + + +class _Transport: + def __init__(self, local=True): + self.auth_identity = None + self.local_desktop = local + self.frames = [] + self.sent = threading.Event() + + def write(self, frame): + self.frames.append(frame) + self.sent.set() + return True + + +REGISTER = { + "protocol_version": 1, + "controller_id": "chrome-extension", + "browser_profile_id": "chrome-default", + "capabilities": ["browser_navigate", "browser_snapshot", "browser_cdp", "unknown.method"], +} + + +def test_local_registration_is_refused_off_the_desktop_transport(monkeypatch): + monkeypatch.setattr("gateway.browser_control_broker.browser_control_local_enabled", lambda config=None: True) + get_browser_control_broker().reset() + assert _rpc("browser.controller.local.register", REGISTER, _Transport(local=False))["error"]["code"] == 4403 + monkeypatch.setattr("gateway.browser_control_broker.browser_control_local_enabled", lambda config=None: False) + assert _rpc("browser.controller.local.register", REGISTER, _Transport())["error"]["code"] == 4403 + assert _rpc("browser.controller.local.register", {**REGISTER, "capabilities": ["browser_cdp"]}, _Transport())["error"]["code"] == 4403 + + +def test_local_gateway_round_trip_and_detach(monkeypatch): + monkeypatch.setattr("gateway.browser_control_broker.browser_control_local_enabled", lambda config=None: True) + broker = get_browser_control_broker() + broker.reset() + transport = _Transport() + try: + registered = _rpc("browser.controller.local.register", REGISTER, transport)["result"] + assert registered["capabilities"] == ["browser_navigate", "browser_snapshot"] + + scope = _lookup(broker, "durable-session") + thread, outcome = _dispatch_async(broker, scope) + assert transport.sent.wait(2) + event = transport.frames[-1] + assert event["method"] == "event" + assert event["params"]["type"] == "browser.controller.command" + assert event["params"]["session_id"] == "durable-session" + command_id = event["params"]["payload"]["command_id"] + + foreign = _rpc("browser.controller.local.result", {"command_id": command_id, "ok": True}, _Transport()) + assert foreign["result"]["accepted"] is False + done = _rpc("browser.controller.local.result", {"command_id": command_id, "ok": True, "result": {"success": True}}, transport) + assert done["result"]["accepted"] is True + thread.join(2) + assert outcome == {"result": {"success": True}} + + assert _rpc("browser.controller.local.detach", {}, transport)["result"]["detached"] is True + assert _lookup(broker, "durable-session") is None + finally: + broker.reset() + + +def test_vision_from_controller_saves_the_jpeg_and_reuses_the_analysis(monkeypatch, tmp_path): + from tools import browser_tool + + monkeypatch.setattr("hermes_constants.get_hermes_dir", lambda *parts: tmp_path) + captured = {} + + def analyze(path, question, annotate, result, *, mime_type="image/png", fallback_warning=None): + captured.update(path=path, question=question, mime=mime_type, bytes=path.read_bytes()) + return {"native": True} + + monkeypatch.setattr(browser_tool, "_analyze_browser_screenshot", analyze) + jpeg = b"\xff\xd8\xff\xe0fixture" + out = browser_tool._browser_vision_from_controller( + {"success": True, "screenshot": base64.b64encode(jpeg).decode(), "mime": "image/jpeg"}, "Ne görünüyor?" + ) + assert out == {"native": True} + assert captured["bytes"] == jpeg and captured["mime"] == "image/jpeg" + assert captured["path"].suffix == ".jpg" and captured["question"] == "Ne görünüyor?" + + bad = json.loads(browser_tool._browser_vision_from_controller({"success": True, "screenshot": "%%%"}, "q")) + assert bad["success"] is False + failed = json.loads(browser_tool._browser_vision_from_controller({"success": False, "error": "Sekme kapandı"}, "q")) + assert failed == {"success": False, "error": "Sekme kapandı"} diff --git a/tests/tools/test_browser_extension_router_wiring.py b/tests/tools/test_browser_extension_router_wiring.py index 8fb7e62..809074b 100644 --- a/tests/tools/test_browser_extension_router_wiring.py +++ b/tests/tools/test_browser_extension_router_wiring.py @@ -19,7 +19,7 @@ def _route_spy(monkeypatch): exercising real routing or a real browser backend.""" calls = [] - def spy(action, args, *, fallback, task_id=None, session_id=None, tool_call_id=None): + def spy(action, args, *, fallback, task_id=None, session_id=None, tool_call_id=None, transform=None): calls.append( { "action": action, @@ -27,6 +27,7 @@ def _route_spy(monkeypatch): "task_id": task_id, "session_id": session_id, "tool_call_id": tool_call_id, + "transform": transform is not None, } ) return fallback() diff --git a/tools/browser_extension_router.py b/tools/browser_extension_router.py index f9cbecc..ce55f53 100644 --- a/tools/browser_extension_router.py +++ b/tools/browser_extension_router.py @@ -45,6 +45,24 @@ from typing import Any, Callable, Dict, Optional logger = logging.getLogger(__name__) +def lane_enabled(transport_family: Optional[str]) -> bool: + """Feature gate for one controller lane. + + The desktop IDE's own browser lane has its own switch + (``browser.extension_control.local_desktop``, on by default); every other + lane keeps the explicit ``browser.extension_control.enabled`` opt-in. + """ + from gateway.browser_control_broker import ( + LOCAL_DESKTOP_TRANSPORT_FAMILY, + browser_control_enabled, + browser_control_local_enabled, + ) + + if transport_family == LOCAL_DESKTOP_TRANSPORT_FAMILY: + return browser_control_local_enabled() + return browser_control_enabled() + + def extension_controller_available(action: str) -> bool: """Whether this request owns one exact controller capable of ``action``. @@ -56,14 +74,9 @@ def extension_controller_available(action: str) -> bool: all fail closed. """ try: - from gateway.browser_control_broker import ( - browser_control_enabled, - get_browser_control_broker, - ) + from gateway.browser_control_broker import get_browser_control_broker from gateway.session_context import get_session_env - if not browser_control_enabled(): - return False session_id = get_session_env("HERMES_SESSION_ID", "") or None principal_id = get_session_env("HERMES_BROWSER_CONTROL_PRINCIPAL", "") or None transport_family = get_session_env( @@ -71,6 +84,8 @@ def extension_controller_available(action: str) -> bool: ) or None if not session_id or not principal_id or not transport_family: return False + if not lane_enabled(transport_family): + return False broker = get_browser_control_broker() scope = broker.scope_for_session( session_id=session_id, @@ -99,6 +114,7 @@ def route_browser_tool( principal_id: Optional[str] = None, transport_family: Optional[str] = None, tool_call_id: Optional[str] = "", + transform: Optional[Callable[[Any], Any]] = None, ) -> Any: """Route one browser action through the extension-control broker. @@ -127,6 +143,11 @@ def route_browser_tool( Hermes callers. tool_call_id: Caller tool-call id forwarded verbatim to ``dispatch``. + transform: + Optional post-processor for a *controller* result only (never for the + fallback). Its return value is the tool result as-is, so it may build + the multimodal envelope (``browser_vision`` turns the controller's + screenshot into the same result the legacy backend returns). Returns ------- @@ -184,6 +205,8 @@ def route_browser_tool( result = broker.dispatch( scope, action=action, arguments=args, tool_call_id=tool_call_id ) + if transform is not None: + return transform(result) if isinstance(result, str): return result return json.dumps(result, ensure_ascii=False) @@ -215,6 +238,7 @@ def routed_browser_handler( principal_id: Optional[str] = None, transport_family: Optional[str] = None, tool_call_id: Optional[str] = None, + transform: Optional[Callable[[Any], Any]] = None, ) -> Any: """Lazy registry-handler route wrapper for ``browser_*`` tools. @@ -225,10 +249,7 @@ def routed_browser_handler( legacy handler runs unchanged. """ try: - from gateway.browser_control_broker import ( - browser_control_enabled, - get_browser_control_broker, - ) + from gateway.browser_control_broker import get_browser_control_broker except Exception as exc: # pragma: no cover - defensive, gateway always present logger.debug( "browser extension router unavailable (%s); using legacy backend", @@ -236,12 +257,6 @@ def routed_browser_handler( ) return fallback() - if not browser_control_enabled(): - return fallback() - - if tool_call_id is None: - tool_call_id = current_tool_call_id() - try: from gateway.session_context import get_session_env @@ -255,6 +270,12 @@ def routed_browser_handler( except Exception: pass + if not lane_enabled(transport_family): + return fallback() + + if tool_call_id is None: + tool_call_id = current_tool_call_id() + return route_browser_tool( action, args, @@ -266,4 +287,5 @@ def routed_browser_handler( principal_id=principal_id, transport_family=transport_family, tool_call_id=tool_call_id, + transform=transform, ) diff --git a/tools/browser_tool.py b/tools/browser_tool.py index ecc72a6..ba17a9c 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -5676,152 +5676,14 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] ), }, ensure_ascii=False) - # NOTE: the full-resolution base64 encode is deliberately deferred. - # The native fast path below sizes its own history-reuse embed via - # _resize_image_for_vision (stat-based quick estimate — no full-res - # encode when oversized), and only the aux-LLM fallback path needs - # the one-shot full-res data URL. - - # Fast path: when native image routing is in effect for the active main - # model, attach the screenshot directly instead of describing it through - # an auxiliary vision LLM. The model inspects the pixels on its next - # turn — no aux call, no information loss. Consistent with vision_analyze. - from tools.vision_tools import ( - _EMBED_MAX_DIMENSION, - _EMBED_TARGET_BYTES, - _build_native_vision_tool_result, - _resize_image_for_vision, - _should_use_native_vision_fast_path, + return _analyze_browser_screenshot( + screenshot_path, + question, + annotate, + result, + fallback_warning=_lp_fallback_warning, ) - if _should_use_native_vision_fast_path(): - # History-reuse cap (#92699): this embed is baked into the tool - # result and re-sent on every later turn, exactly like - # vision_analyze's native path — apply the same proactive resize - # so full-res screenshots can't enter immutable history uncapped. - # The helper's internal stat/dimension quick-estimate skips the - # resize (and encodes directly) when the screenshot is already - # under both caps, so no full-res base64 is built just to be - # thrown away. Fail-open: without Pillow it falls back to the - # raw bytes and the compressor's keep-newest pass still retires - # stale embeds. - data_url = _resize_image_for_vision( - screenshot_path, - mime_type="image/png", - max_base64_bytes=_EMBED_TARGET_BYTES, - max_dimension=_EMBED_MAX_DIMENSION, - force_jpeg=True, - ) - native_result = _build_native_vision_tool_result( - image_url=str(screenshot_path), - question=question, - image_data_url=data_url, - image_size_bytes=screenshot_path.stat().st_size, - ) - meta = native_result.setdefault("meta", {}) - meta["screenshot_path"] = str(screenshot_path) - if _lp_fallback_warning: - meta["fallback_warning"] = _lp_fallback_warning - if annotate and result.get("data", {}).get("annotations"): - meta["annotations"] = result["data"]["annotations"] - native_result["text_summary"] = ( - f"{native_result.get('text_summary', '')} " - f"Screenshot path: {screenshot_path}" - ).strip() - return native_result - - vision_prompt = ( - f"You are analyzing a screenshot of a web browser.\n\n" - f"User's question: {question}\n\n" - f"Provide a detailed and helpful answer based on what you see in the screenshot. " - f"If there are interactive elements, describe them. If there are verification challenges " - f"or CAPTCHAs, describe what type they are and what action might be needed. " - f"Focus on answering the user's specific question." - ) - - # Aux-LLM path: one-shot analysis, not baked into history — encode at - # full resolution here (the pre-existing 5 MB oversize guard below - # still applies). - _screenshot_bytes = screenshot_path.read_bytes() - _screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii") - data_url = f"data:image/png;base64,{_screenshot_b64}" - - # Use the centralized LLM router - vision_model = _get_vision_model() - logger.debug("browser_vision: analysing screenshot (%d bytes)", - len(_screenshot_bytes)) - - # Read vision timeout/temperature from config (auxiliary.vision.*). - # Local vision models (llama.cpp, ollama) can take well over 30s for - # screenshot analysis, so the default timeout must be generous. - vision_timeout = 120.0 - vision_temperature = 0.1 - try: - from hermes_cli.config import load_config - _cfg = load_config() - _vision_cfg = cfg_get(_cfg, "auxiliary", "vision", default={}) - _vt = _vision_cfg.get("timeout") - if _vt is not None: - vision_timeout = float(_vt) - _vtemp = _vision_cfg.get("temperature") - if _vtemp is not None: - vision_temperature = float(_vtemp) - except Exception: - pass - - call_kwargs = { - "task": "vision", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": vision_prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - } - ], - "temperature": vision_temperature, - "timeout": vision_timeout, - } - if vision_model: - call_kwargs["model"] = vision_model - # Try full-size screenshot; on size-related rejection, downscale and retry. - try: - response = _lazy_call_llm(**call_kwargs) - except Exception as _api_err: - from tools.vision_tools import ( - _is_image_size_error, _resize_image_for_vision, _RESIZE_TARGET_BYTES, - ) - if (_is_image_size_error(_api_err) - and len(data_url) > _RESIZE_TARGET_BYTES): - logger.info( - "Vision API rejected screenshot (%.1f MB); " - "auto-resizing to ~%.0f MB and retrying...", - len(data_url) / (1024 * 1024), - _RESIZE_TARGET_BYTES / (1024 * 1024), - ) - data_url = _resize_image_for_vision( - screenshot_path, mime_type="image/png") - call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url - response = _lazy_call_llm(**call_kwargs) - else: - raise - - analysis = (response.choices[0].message.content or "").strip() - # Redact secrets the vision LLM may have read from the screenshot. - from agent.redact import redact_sensitive_text - analysis = redact_sensitive_text(analysis) - response_data = { - "success": True, - "analysis": analysis or "Vision analysis returned no content.", - "screenshot_path": str(screenshot_path), - } - _copy_fallback_warning(response_data, result) - # Include annotation data if annotated screenshot was taken - if annotate and result.get("data", {}).get("annotations"): - response_data["annotations"] = result["data"]["annotations"] - return json.dumps(response_data, ensure_ascii=False) - except Exception as e: # Keep the screenshot if it was captured successfully — the failure is # in the LLM vision analysis, not the capture. Deleting a valid @@ -5836,6 +5698,243 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] return json.dumps(error_info, ensure_ascii=False) +#: Largest decoded screenshot accepted from a browser-extension controller. +_CONTROLLER_SCREENSHOT_MAX_BYTES = 8 * 1024 * 1024 + + +def _browser_vision_from_controller( + controller_result: Any, question: str, annotate: bool = False +) -> Union[str, Dict[str, Any]]: + """Turn an extension controller's screenshot into a browser_vision result. + + The controller (the user's own browser) only captures the visible tab: + ``{"success": true, "screenshot": , "mime": "image/jpeg", ...}``. + Saving the image and the native/auxiliary vision analysis stay here, so + the model receives exactly what the legacy backend would return. + """ + import base64 + import uuid as uuid_mod + from hermes_constants import get_hermes_dir + + data = controller_result + if isinstance(data, str): + try: + data = json.loads(data) + except (TypeError, ValueError): + return data + if not isinstance(data, dict): + return json.dumps( + {"success": False, "error": "Browser returned no screenshot."}, + ensure_ascii=False, + ) + if data.get("success") is not True or not isinstance(data.get("screenshot"), str): + return json.dumps( + { + "success": False, + "error": str(data.get("error") or "Browser returned no screenshot."), + }, + ensure_ascii=False, + ) + mime = "image/png" if data.get("mime") == "image/png" else "image/jpeg" + try: + raw = base64.b64decode(data["screenshot"], validate=True) + except (ValueError, TypeError): + return json.dumps( + {"success": False, "error": "Browser returned an unreadable screenshot."}, + ensure_ascii=False, + ) + if not raw or len(raw) > _CONTROLLER_SCREENSHOT_MAX_BYTES: + return json.dumps( + {"success": False, "error": "Browser screenshot is empty or too large."}, + ensure_ascii=False, + ) + screenshots_dir = get_hermes_dir("cache/screenshots", "browser_screenshots") + screenshots_dir.mkdir(parents=True, exist_ok=True) + _cleanup_old_screenshots(screenshots_dir, max_age_hours=24) + suffix = ".png" if mime == "image/png" else ".jpg" + screenshot_path = screenshots_dir / f"browser_screenshot_{uuid_mod.uuid4().hex}{suffix}" + screenshot_path.write_bytes(raw) + capture = {"success": True, "data": {"path": str(screenshot_path)}} + annotations = data.get("annotations") + if annotate and isinstance(annotations, list): + capture["data"]["annotations"] = annotations + try: + return _analyze_browser_screenshot( + screenshot_path, question, annotate, capture, mime_type=mime + ) + except Exception as e: + logger.warning("browser_vision (extension) failed: %s", e, exc_info=True) + return json.dumps( + { + "success": False, + "error": f"Error during vision analysis: {str(e)}", + "screenshot_path": str(screenshot_path), + "note": "Screenshot was captured but vision analysis failed. You can still share it via MEDIA:.", + }, + ensure_ascii=False, + ) + + +def _analyze_browser_screenshot( + screenshot_path: Path, + question: str, + annotate: bool, + result: Dict[str, Any], + *, + mime_type: str = "image/png", + fallback_warning: Optional[str] = None, +) -> Union[str, Dict[str, Any]]: + """Native-attach or auxiliary-LLM analysis of one saved browser screenshot.""" + import base64 + + _lp_fallback_warning = fallback_warning + # NOTE: the full-resolution base64 encode is deliberately deferred. + # The native fast path below sizes its own history-reuse embed via + # _resize_image_for_vision (stat-based quick estimate — no full-res + # encode when oversized), and only the aux-LLM fallback path needs + # the one-shot full-res data URL. + + # Fast path: when native image routing is in effect for the active main + # model, attach the screenshot directly instead of describing it through + # an auxiliary vision LLM. The model inspects the pixels on its next + # turn — no aux call, no information loss. Consistent with vision_analyze. + from tools.vision_tools import ( + _EMBED_MAX_DIMENSION, + _EMBED_TARGET_BYTES, + _build_native_vision_tool_result, + _resize_image_for_vision, + _should_use_native_vision_fast_path, + ) + + if _should_use_native_vision_fast_path(): + # History-reuse cap (#92699): this embed is baked into the tool + # result and re-sent on every later turn, exactly like + # vision_analyze's native path — apply the same proactive resize + # so full-res screenshots can't enter immutable history uncapped. + # The helper's internal stat/dimension quick-estimate skips the + # resize (and encodes directly) when the screenshot is already + # under both caps, so no full-res base64 is built just to be + # thrown away. Fail-open: without Pillow it falls back to the + # raw bytes and the compressor's keep-newest pass still retires + # stale embeds. + data_url = _resize_image_for_vision( + screenshot_path, + mime_type=mime_type, + max_base64_bytes=_EMBED_TARGET_BYTES, + max_dimension=_EMBED_MAX_DIMENSION, + force_jpeg=True, + ) + native_result = _build_native_vision_tool_result( + image_url=str(screenshot_path), + question=question, + image_data_url=data_url, + image_size_bytes=screenshot_path.stat().st_size, + ) + meta = native_result.setdefault("meta", {}) + meta["screenshot_path"] = str(screenshot_path) + if _lp_fallback_warning: + meta["fallback_warning"] = _lp_fallback_warning + if annotate and result.get("data", {}).get("annotations"): + meta["annotations"] = result["data"]["annotations"] + native_result["text_summary"] = ( + f"{native_result.get('text_summary', '')} " + f"Screenshot path: {screenshot_path}" + ).strip() + return native_result + + vision_prompt = ( + f"You are analyzing a screenshot of a web browser.\n\n" + f"User's question: {question}\n\n" + f"Provide a detailed and helpful answer based on what you see in the screenshot. " + f"If there are interactive elements, describe them. If there are verification challenges " + f"or CAPTCHAs, describe what type they are and what action might be needed. " + f"Focus on answering the user's specific question." + ) + + # Aux-LLM path: one-shot analysis, not baked into history — encode at + # full resolution here (the pre-existing 5 MB oversize guard below + # still applies). + _screenshot_bytes = screenshot_path.read_bytes() + _screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii") + data_url = f"data:{mime_type};base64,{_screenshot_b64}" + + # Use the centralized LLM router + vision_model = _get_vision_model() + logger.debug("browser_vision: analysing screenshot (%d bytes)", + len(_screenshot_bytes)) + + # Read vision timeout/temperature from config (auxiliary.vision.*). + # Local vision models (llama.cpp, ollama) can take well over 30s for + # screenshot analysis, so the default timeout must be generous. + vision_timeout = 120.0 + vision_temperature = 0.1 + try: + from hermes_cli.config import load_config + _cfg = load_config() + _vision_cfg = cfg_get(_cfg, "auxiliary", "vision", default={}) + _vt = _vision_cfg.get("timeout") + if _vt is not None: + vision_timeout = float(_vt) + _vtemp = _vision_cfg.get("temperature") + if _vtemp is not None: + vision_temperature = float(_vtemp) + except Exception: + pass + + call_kwargs = { + "task": "vision", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": vision_prompt}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + } + ], + "temperature": vision_temperature, + "timeout": vision_timeout, + } + if vision_model: + call_kwargs["model"] = vision_model + # Try full-size screenshot; on size-related rejection, downscale and retry. + try: + response = _lazy_call_llm(**call_kwargs) + except Exception as _api_err: + from tools.vision_tools import ( + _is_image_size_error, _resize_image_for_vision, _RESIZE_TARGET_BYTES, + ) + if (_is_image_size_error(_api_err) + and len(data_url) > _RESIZE_TARGET_BYTES): + logger.info( + "Vision API rejected screenshot (%.1f MB); " + "auto-resizing to ~%.0f MB and retrying...", + len(data_url) / (1024 * 1024), + _RESIZE_TARGET_BYTES / (1024 * 1024), + ) + data_url = _resize_image_for_vision( + screenshot_path, mime_type=mime_type) + call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url + response = _lazy_call_llm(**call_kwargs) + else: + raise + + analysis = (response.choices[0].message.content or "").strip() + # Redact secrets the vision LLM may have read from the screenshot. + from agent.redact import redact_sensitive_text + analysis = redact_sensitive_text(analysis) + response_data = { + "success": True, + "analysis": analysis or "Vision analysis returned no content.", + "screenshot_path": str(screenshot_path), + } + _copy_fallback_warning(response_data, result) + # Include annotation data if annotated screenshot was taken + if annotate and result.get("data", {}).get("annotations"): + response_data["annotations"] = result["data"]["annotations"] + return json.dumps(response_data, ensure_ascii=False) + + def _cleanup_old_screenshots(screenshots_dir, max_age_hours=24): """Remove browser screenshots older than max_age_hours to prevent disk bloat. @@ -6381,7 +6480,10 @@ def check_browser_vision_requirements() -> bool: fails at call time with a cryptic provider-side error like ``unknown variant `image_url`, expected `text``` (issue #31179). """ - if not check_browser_requirements(): + if not ( + check_browser_requirements() + or extension_controller_available("browser_vision") + ): return False try: from tools.vision_tools import check_vision_requirements @@ -6602,7 +6704,7 @@ registry.register( fallback=lambda: browser_get_images(task_id=kw.get("task_id")), **_browser_router_kw(kw), ), - check_fn=check_browser_requirements, + check_fn=lambda: check_browser_routed_requirements("browser_get_images"), emoji="🖼️", ) registry.register( @@ -6613,6 +6715,9 @@ registry.register( "browser_vision", args, fallback=lambda: browser_vision(question=args.get("question", ""), annotate=args.get("annotate", False), task_id=kw.get("task_id")), + transform=lambda result: _browser_vision_from_controller( + result, args.get("question", ""), bool(args.get("annotate", False)) + ), **_browser_router_kw(kw), ), check_fn=check_browser_vision_requirements, diff --git a/tui_gateway/methods_browser_control.py b/tui_gateway/methods_browser_control.py index abd7afa..24cf45d 100644 --- a/tui_gateway/methods_browser_control.py +++ b/tui_gateway/methods_browser_control.py @@ -38,6 +38,8 @@ import logging from gateway.browser_control_broker import ( BROWSER_CONTROL_PROTOCOL_VERSION, + LOCAL_DESKTOP_PRINCIPAL, + LOCAL_DESKTOP_TRANSPORT_FAMILY, browser_control_protocol_supported, filter_browser_control_capabilities, ) @@ -58,9 +60,17 @@ method = _registry.method #: transport can never address a dashboard controller (and vice versa). _CLOUD_TRANSPORT_FAMILY = "cloud-ticket-ws" +#: Local desktop lane: the AITURK Chrome extension reaches this backend only +#: through the desktop IDE's native bridge, which authenticates with the +#: loopback session token (``WSTransport.local_desktop``). +_LOCAL_TRANSPORT_FAMILY = LOCAL_DESKTOP_TRANSPORT_FAMILY +_LOCAL_PRINCIPAL = LOCAL_DESKTOP_PRINCIPAL + #: JSON-RPC error code for identity / session / flag denials (forbidden). _ERR_FORBIDDEN = 4403 +_ID_LIMIT = 200 + def _is_authenticated_identity(identity: object) -> bool: """True for a server-minted, non-internal ``{user_id, provider}`` identity.""" @@ -377,6 +387,130 @@ def _( return _ok(rid, {"detached": True}) +def _is_local_desktop(transport: object) -> bool: + return getattr(transport, "local_desktop", False) is True + + +@method("browser.controller.local.register") +def _( + rid, + params: dict, + _family=_LOCAL_TRANSPORT_FAMILY, + _principal=_LOCAL_PRINCIPAL, + _protocol_supported=browser_control_protocol_supported, + _protocol_version=BROWSER_CONTROL_PROTOCOL_VERSION, + _filter_capabilities=filter_browser_control_capabilities, + _forbidden=_ERR_FORBIDDEN, + _local=_is_local_desktop, + _event_writer=_broker_event_writer, + _id_limit=_ID_LIMIT, +) -> dict: + """Offer the desktop user's own browser to every desktop session. + + Only the loopback desktop transport (the IDE's native Chrome bridge) may + register, and only while ``browser.extension_control.local_desktop`` is + on. The controller becomes a lane template: each desktop session that + uses a browser tool gets its own exact scope on first use, and its frames + carry that session id. Registering again from the same transport + refreshes it; closing the transport removes it, so sessions go back to + their normal browser backend. + """ + from gateway import browser_control_broker + + transport = current_transport() + if not _local(transport): + return _err( + rid, _forbidden, "local browser controller requires the desktop transport" + ) + if not browser_control_broker.browser_control_local_enabled(): + return _err(rid, _forbidden, "browser.extension_control.local_desktop is off") + if not _protocol_supported(params.get("protocol_version")): + return _err( + rid, + _forbidden, + f"unsupported browser-control protocol version; expected {_protocol_version}", + ) + controller_id = str(params.get("controller_id") or "").strip() + browser_profile_id = str(params.get("browser_profile_id") or "").strip() + profile_id = str(params.get("profile") or "local").strip() or "local" + if ( + not controller_id + or not browser_profile_id + or max(len(controller_id), len(browser_profile_id), len(profile_id)) > _id_limit + ): + return _err(rid, _forbidden, "controller_id and browser_profile_id are required") + capabilities = _filter_capabilities(params.get("capabilities")) + if not capabilities: + return _err(rid, _forbidden, "no permitted controller capabilities requested") + template = browser_control_broker.ControllerScope( + principal_id=_principal, + profile_id=profile_id, + session_id=None, + controller_id=controller_id, + browser_profile_id=browser_profile_id, + transport_family=_family, + capabilities=capabilities, + ) + broker = browser_control_broker.get_browser_control_broker() + broker.attach_lane_template( + template, + lambda session_id: _event_writer(transport, session_id), + owner=transport, + ) + return _ok( + rid, + { + "registered": True, + "transport_family": _family, + "capabilities": sorted(capabilities), + }, + ) + + +@method("browser.controller.local.result") +def _(rid, params: dict, _forbidden=_ERR_FORBIDDEN, _local=_is_local_desktop) -> dict: + """Deliver one local controller result; only its own transport may.""" + from gateway import browser_control_broker + + transport = current_transport() + if not _local(transport): + return _err(rid, _forbidden, "local browser controller requires the desktop transport") + command_id = str(params.get("command_id") or "") + if not command_id: + return _err(rid, _forbidden, "command_id required") + ok = params.get("ok") is True + accepted = browser_control_broker.get_browser_control_broker().complete_for_owner( + command_id, + owner=transport, + ok=ok, + result=params.get("result") if ok else params.get("error"), + ) + return _ok(rid, {"accepted": accepted}) + + +@method("browser.controller.local.detach") +def _( + rid, + params: dict, + _family=_LOCAL_TRANSPORT_FAMILY, + _principal=_LOCAL_PRINCIPAL, + _forbidden=_ERR_FORBIDDEN, + _local=_is_local_desktop, +) -> dict: + """Stop offering the user's browser; pending commands are cancelled.""" + from gateway import browser_control_broker + + transport = current_transport() + if not _local(transport): + return _err(rid, _forbidden, "local browser controller requires the desktop transport") + detached = browser_control_broker.get_browser_control_broker().detach_lane_template( + principal_id=_principal, + transport_family=_family, + owner=transport, + ) + return _ok(rid, {"detached": detached}) + + def register(server) -> None: """Bind this module's handlers onto ``server``'s globals and registry.""" _registry.install(server) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index f50a001..aa814c9 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -4848,6 +4848,17 @@ def _set_session_context( browser_control_transport_family = ( _methods_browser_control._CLOUD_TRANSPORT_FAMILY ) + elif getattr(transport, "local_desktop", False) is True: + # The desktop IDE's own browser (AITURK Chrome) serves + # desktop sessions once its extension registers; until + # then the lane is unregistered and the normal browser + # backend is used unchanged. + browser_control_principal = ( + _methods_browser_control._LOCAL_PRINCIPAL + ) + browser_control_transport_family = ( + _methods_browser_control._LOCAL_TRANSPORT_FAMILY + ) break return set_session_vars( session_key=session_key, diff --git a/tui_gateway/ws.py b/tui_gateway/ws.py index e0b8bbe..4572384 100644 --- a/tui_gateway/ws.py +++ b/tui_gateway/ws.py @@ -129,10 +129,15 @@ class WSTransport: *, peer: str = "unknown", auth_identity: dict | None = None, + local_desktop: bool = False, ) -> None: self._ws = ws self._loop = loop self._peer = peer + #: True only for the loopback desktop's session-token credential (the + #: desktop IDE and its own bridges). Never set from RPC params; it is + #: the authority for the local desktop browser-controller lane. + self.local_desktop = local_desktop is True #: Server-verified identity carried from the WS-upgrade credential #: (dashboard ticket / internal credential) — stamped by #: ``hermes_cli.web_server._ws_auth_reason`` onto the WS object and @@ -369,6 +374,7 @@ async def handle_ws( *, auth_identity: dict | None = None, subprotocol: str | None = None, + local_desktop: bool = False, ) -> None: """Run one WebSocket session. Wire-compatible with ``tui_gateway.entry``. @@ -406,6 +412,7 @@ async def handle_ws( asyncio.get_running_loop(), peer=peer, auth_identity=auth_identity, + local_desktop=local_desktop, ) # resolve_skin() reads config + initializes the skin engine — diff --git a/website/docs/user-guide/features/api-server.md b/website/docs/user-guide/features/api-server.md index 36b6e1c..144d913 100644 --- a/website/docs/user-guide/features/api-server.md +++ b/website/docs/user-guide/features/api-server.md @@ -284,6 +284,7 @@ protocol version, transport names, and the exact capability allowlist: controller.noop browser_back browser_click +browser_get_images browser_navigate browser_press browser_screenshot @@ -292,11 +293,25 @@ browser_snapshot browser_tab_activate browser_tabs browser_type +browser_vision ``` Requested capabilities outside that list are filtered out. Raw CDP, arbitrary -script evaluation, console access, uploads, image extraction, and vision are not -part of the controller protocol. +script evaluation, console access and uploads are not part of the controller +protocol. For `browser_vision` the controller only returns a capture of the +visible tab; Hermes saves it and runs the usual vision analysis itself. + +### Desktop browser lane + +The desktop app's own browser extension (AITURK Chrome) registers through the +app's native bridge with `browser.controller.local.register` on the loopback +desktop WebSocket. One registration serves every desktop session of that +backend; each session still gets its own exact controller scope on first use, +and command events carry that session id. The lane has its own switch, +`browser.extension_control.local_desktop` (on by default), independent of +`enabled`. When the extension disconnects or calls +`browser.controller.local.detach`, pending commands are cancelled and desktop +sessions return to their normal browser backend. When a request has no bound controller identity, or when the feature is disabled, Hermes preserves the existing browser backend. Once the gateway binds a