feat(chrome): let the IDE agent use the user's Chrome and beta.42
Server CI and Windows package / verify (push) Successful in 3m33s
Server CI and Windows package / windows-package (push) Successful in 7m36s

The AITURK Chrome extension now registers the user's Chrome as the desktop
backend's browser controller, the way the ChatGPT extension lets its agent
work in the user's browser. The agent's existing browser tools (navigate,
snapshot, click, type, scroll, back, press, get_images, vision) run in its
own background tabs grouped as "AITURK", or a tab the user hands over.

Backend: a local desktop lane in the browser-control broker. One controller
registered from the loopback desktop transport is offered to every desktop
session as a lane template; each session still gets its own exact scope on
first use. The lane has its own switch (browser.extension_control.local_desktop,
on by default); with no extension registered, or after it disconnects, the
normal browser backend runs unchanged. browser_vision turns the extension's
capture into the usual native/auxiliary vision result.

Bridge: forwards controller commands only to a connection that registered,
refuses unknown or oversized commands, relays results.

Extension: service-worker controller with its own native connection,
trusted chrome.debugger input (press/release only: a hover move waits 5 s on
background tabs), refusal of password and payment-card fields and of
browser, extension, file and Web Store pages, panel switch, Stop, tab
hand-over, recent actions, Alt+Shift+A and right-click "send to AITURK".
Cancelling Chrome's debugging bar stops the agent. Existing panel flows and
privacy rules are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
yilsemandClaude Opus 5.5 committed 2026-10-15 08:38:52 +03:00
1 parent 5c76c24c9c
commit 86a6621936
32 files changed
+2755 -195

No files matched your search

+68 -1
View File
@@ -31,12 +31,33 @@ export function publicMessages(rows) {
return messages
}
// Browser actions the extension may offer the IDE agent. The backend applies
// its own allowlist again; this one keeps unknown names off the wire.
export const BROWSER_CAPABILITIES = Object.freeze(['browser_navigate', 'browser_snapshot', 'browser_click',
'browser_type', 'browser_scroll', 'browser_back', 'browser_press', 'browser_get_images', 'browser_vision'])
const COMMAND_ID = /^[a-f0-9]{16,64}$/
export function validateRequest(message) {
if (!message || typeof message.id !== 'string' || message.id.length > 80 ||
!['status', 'sessions.list', 'session.open', 'session.history', 'message.send', 'message.stop'].includes(message.action)) {
!['status', 'sessions.list', 'session.open', 'session.history', 'message.send', 'message.stop',
'browser.register', 'browser.result'].includes(message.action)) {
throw new Error('Desteklenmeyen eklenti isteği.')
}
const params = message.params || {}
if (message.action === 'browser.register') {
if (typeof params.enabled !== 'boolean') throw new Error('Tarayıcı denetimi ayarı geçersiz.')
if (params.enabled && (!Array.isArray(params.capabilities) || !params.capabilities.length ||
params.capabilities.some(name => !BROWSER_CAPABILITIES.includes(name)) ||
!/^[a-zA-Z0-9_.:-]{1,120}$/.test(params.controllerId || '') ||
!/^[a-zA-Z0-9_.:-]{1,120}$/.test(params.browserProfileId || ''))) {
throw new Error('Tarayıcı denetimi kaydı geçersiz.')
}
}
if (message.action === 'browser.result' &&
(typeof params.commandId !== 'string' || !COMMAND_ID.test(params.commandId) || typeof params.ok !== 'boolean' ||
(!params.ok && (typeof params.error !== 'string' || params.error.length > 4000)))) {
throw new Error('Tarayıcı komut sonucu geçersiz.')
}
if (['session.open', 'session.history', 'message.send', 'message.stop'].includes(message.action) &&
(typeof params.sessionId !== 'string' || !/^[a-zA-Z0-9_.:-]{1,160}$/.test(params.sessionId))) {
throw new Error('Geçersiz konuşma kimliği.')
@@ -57,6 +78,8 @@ export class ChromeGatewayClient {
this.nextId = 0
this.selected = null
this.closed = false
// True once this connection offered the user's browser to the IDE agent.
this.controller = false
}
async connect() {
@@ -92,6 +115,8 @@ export class ChromeGatewayClient {
}
this.pending.clear()
this.selected = null
// The backend drops a local browser controller with its transport.
this.controller = false
if (!this.closed) this.emit({ event: 'disconnected', message: 'IDE bağlantısı kesildi. Yeniden bağlanın.' })
})
await new Promise((resolve, reject) => {
@@ -135,6 +160,10 @@ export class ChromeGatewayClient {
}
onEvent(event) {
if (this.controller && ['browser.controller.command', 'browser.controller.cancel'].includes(event?.type)) {
this.onControllerEvent(event)
return
}
const selected = this.selected
if (!selected || event?.session_id !== selected.runtimeId) return
const payload = event.payload || {}
@@ -152,6 +181,26 @@ export class ChromeGatewayClient {
}
}
onControllerEvent(event) {
const payload = event.payload || {}
const commandId = payload.command_id
if (typeof commandId !== 'string' || !COMMAND_ID.test(commandId)) return
if (event.type === 'browser.controller.cancel') {
this.emit({ event: 'browser.cancel', commandId })
return
}
const refuse = error => void this.rpc('browser.controller.local.result',
{ command_id: commandId, ok: false, error }).catch(() => {})
if (!BROWSER_CAPABILITIES.includes(payload.action)) return refuse('Chrome bu tarayıcı işlemini desteklemiyor.')
const args = payload.arguments && typeof payload.arguments === 'object' && !Array.isArray(payload.arguments)
? payload.arguments : {}
// A frame over the native limit would drop the whole connection; refuse
// the one oversized command instead.
if (Buffer.byteLength(JSON.stringify(args)) > 256 * 1024) return refuse('Tarayıcı komutu çok büyük.')
this.emit({ event: 'browser.command', commandId, action: payload.action, arguments: args,
sessionId: typeof event.session_id === 'string' ? event.session_id.slice(0, 160) : '' })
}
async handle(message) {
const params = validateRequest(message)
await this.connect()
@@ -193,6 +242,24 @@ export class ChromeGatewayClient {
}
case 'session.history':
return this.history(params.sessionId)
case 'browser.register': {
if (!params.enabled) {
this.controller = false
const result = await this.rpc('browser.controller.local.detach', {})
return { registered: false, detached: Boolean(result?.detached) }
}
const result = await this.rpc('browser.controller.local.register', { protocol_version: 1,
controller_id: params.controllerId, browser_profile_id: params.browserProfileId,
capabilities: params.capabilities, profile: this.backend.profile || 'default' })
this.controller = true
return { registered: true, capabilities: Array.isArray(result?.capabilities) ? result.capabilities : [] }
}
case 'browser.result': {
const result = await this.rpc('browser.controller.local.result', params.ok
? { command_id: params.commandId, ok: true, result: params.result ?? null }
: { command_id: params.commandId, ok: false, error: params.error })
return { accepted: Boolean(result?.accepted) }
}
case 'message.send':
case 'message.stop': {
if (this.selected?.storedId !== params.sessionId) throw new Error('Önce bu konuşmayı seçin.')
+1 -1
View File
@@ -2,7 +2,7 @@
"name": "aiturk-ide",
"productName": "AITURK IDE",
"private": true,
"version": "1.0.0-beta.41",
"version": "1.0.0-beta.42",
"description": "TurkServis resmi yapay zekâ geliştirme ortamı. Hermes Agent tabanlıdır.",
"author": "AITURK / TurkServis",
"repository": {