feat(chrome): let the IDE agent use the user's Chrome and beta.42
The AITURK Chrome extension now registers the user's Chrome as the desktop backend's browser controller, the way the ChatGPT extension lets its agent work in the user's browser. The agent's existing browser tools (navigate, snapshot, click, type, scroll, back, press, get_images, vision) run in its own background tabs grouped as "AITURK", or a tab the user hands over. Backend: a local desktop lane in the browser-control broker. One controller registered from the loopback desktop transport is offered to every desktop session as a lane template; each session still gets its own exact scope on first use. The lane has its own switch (browser.extension_control.local_desktop, on by default); with no extension registered, or after it disconnects, the normal browser backend runs unchanged. browser_vision turns the extension's capture into the usual native/auxiliary vision result. Bridge: forwards controller commands only to a connection that registered, refuses unknown or oversized commands, relays results. Extension: service-worker controller with its own native connection, trusted chrome.debugger input (press/release only: a hover move waits 5 s on background tabs), refusal of password and payment-card fields and of browser, extension, file and Web Store pages, panel switch, Stop, tab hand-over, recent actions, Alt+Shift+A and right-click "send to AITURK". Cancelling Chrome's debugging bar stops the agent. Existing panel flows and privacy rules are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5c76c24c9c
commit
86a6621936
32 files changed
+2755
-195
No files matched your search
@@ -31,12 +31,33 @@ export function publicMessages(rows) {
|
||||
return messages
|
||||
}
|
||||
|
||||
// Browser actions the extension may offer the IDE agent. The backend applies
|
||||
// its own allowlist again; this one keeps unknown names off the wire.
|
||||
export const BROWSER_CAPABILITIES = Object.freeze(['browser_navigate', 'browser_snapshot', 'browser_click',
|
||||
'browser_type', 'browser_scroll', 'browser_back', 'browser_press', 'browser_get_images', 'browser_vision'])
|
||||
const COMMAND_ID = /^[a-f0-9]{16,64}$/
|
||||
|
||||
export function validateRequest(message) {
|
||||
if (!message || typeof message.id !== 'string' || message.id.length > 80 ||
|
||||
!['status', 'sessions.list', 'session.open', 'session.history', 'message.send', 'message.stop'].includes(message.action)) {
|
||||
!['status', 'sessions.list', 'session.open', 'session.history', 'message.send', 'message.stop',
|
||||
'browser.register', 'browser.result'].includes(message.action)) {
|
||||
throw new Error('Desteklenmeyen eklenti isteği.')
|
||||
}
|
||||
const params = message.params || {}
|
||||
if (message.action === 'browser.register') {
|
||||
if (typeof params.enabled !== 'boolean') throw new Error('Tarayıcı denetimi ayarı geçersiz.')
|
||||
if (params.enabled && (!Array.isArray(params.capabilities) || !params.capabilities.length ||
|
||||
params.capabilities.some(name => !BROWSER_CAPABILITIES.includes(name)) ||
|
||||
!/^[a-zA-Z0-9_.:-]{1,120}$/.test(params.controllerId || '') ||
|
||||
!/^[a-zA-Z0-9_.:-]{1,120}$/.test(params.browserProfileId || ''))) {
|
||||
throw new Error('Tarayıcı denetimi kaydı geçersiz.')
|
||||
}
|
||||
}
|
||||
if (message.action === 'browser.result' &&
|
||||
(typeof params.commandId !== 'string' || !COMMAND_ID.test(params.commandId) || typeof params.ok !== 'boolean' ||
|
||||
(!params.ok && (typeof params.error !== 'string' || params.error.length > 4000)))) {
|
||||
throw new Error('Tarayıcı komut sonucu geçersiz.')
|
||||
}
|
||||
if (['session.open', 'session.history', 'message.send', 'message.stop'].includes(message.action) &&
|
||||
(typeof params.sessionId !== 'string' || !/^[a-zA-Z0-9_.:-]{1,160}$/.test(params.sessionId))) {
|
||||
throw new Error('Geçersiz konuşma kimliği.')
|
||||
@@ -57,6 +78,8 @@ export class ChromeGatewayClient {
|
||||
this.nextId = 0
|
||||
this.selected = null
|
||||
this.closed = false
|
||||
// True once this connection offered the user's browser to the IDE agent.
|
||||
this.controller = false
|
||||
}
|
||||
|
||||
async connect() {
|
||||
@@ -92,6 +115,8 @@ export class ChromeGatewayClient {
|
||||
}
|
||||
this.pending.clear()
|
||||
this.selected = null
|
||||
// The backend drops a local browser controller with its transport.
|
||||
this.controller = false
|
||||
if (!this.closed) this.emit({ event: 'disconnected', message: 'IDE bağlantısı kesildi. Yeniden bağlanın.' })
|
||||
})
|
||||
await new Promise((resolve, reject) => {
|
||||
@@ -135,6 +160,10 @@ export class ChromeGatewayClient {
|
||||
}
|
||||
|
||||
onEvent(event) {
|
||||
if (this.controller && ['browser.controller.command', 'browser.controller.cancel'].includes(event?.type)) {
|
||||
this.onControllerEvent(event)
|
||||
return
|
||||
}
|
||||
const selected = this.selected
|
||||
if (!selected || event?.session_id !== selected.runtimeId) return
|
||||
const payload = event.payload || {}
|
||||
@@ -152,6 +181,26 @@ export class ChromeGatewayClient {
|
||||
}
|
||||
}
|
||||
|
||||
onControllerEvent(event) {
|
||||
const payload = event.payload || {}
|
||||
const commandId = payload.command_id
|
||||
if (typeof commandId !== 'string' || !COMMAND_ID.test(commandId)) return
|
||||
if (event.type === 'browser.controller.cancel') {
|
||||
this.emit({ event: 'browser.cancel', commandId })
|
||||
return
|
||||
}
|
||||
const refuse = error => void this.rpc('browser.controller.local.result',
|
||||
{ command_id: commandId, ok: false, error }).catch(() => {})
|
||||
if (!BROWSER_CAPABILITIES.includes(payload.action)) return refuse('Chrome bu tarayıcı işlemini desteklemiyor.')
|
||||
const args = payload.arguments && typeof payload.arguments === 'object' && !Array.isArray(payload.arguments)
|
||||
? payload.arguments : {}
|
||||
// A frame over the native limit would drop the whole connection; refuse
|
||||
// the one oversized command instead.
|
||||
if (Buffer.byteLength(JSON.stringify(args)) > 256 * 1024) return refuse('Tarayıcı komutu çok büyük.')
|
||||
this.emit({ event: 'browser.command', commandId, action: payload.action, arguments: args,
|
||||
sessionId: typeof event.session_id === 'string' ? event.session_id.slice(0, 160) : '' })
|
||||
}
|
||||
|
||||
async handle(message) {
|
||||
const params = validateRequest(message)
|
||||
await this.connect()
|
||||
@@ -193,6 +242,24 @@ export class ChromeGatewayClient {
|
||||
}
|
||||
case 'session.history':
|
||||
return this.history(params.sessionId)
|
||||
case 'browser.register': {
|
||||
if (!params.enabled) {
|
||||
this.controller = false
|
||||
const result = await this.rpc('browser.controller.local.detach', {})
|
||||
return { registered: false, detached: Boolean(result?.detached) }
|
||||
}
|
||||
const result = await this.rpc('browser.controller.local.register', { protocol_version: 1,
|
||||
controller_id: params.controllerId, browser_profile_id: params.browserProfileId,
|
||||
capabilities: params.capabilities, profile: this.backend.profile || 'default' })
|
||||
this.controller = true
|
||||
return { registered: true, capabilities: Array.isArray(result?.capabilities) ? result.capabilities : [] }
|
||||
}
|
||||
case 'browser.result': {
|
||||
const result = await this.rpc('browser.controller.local.result', params.ok
|
||||
? { command_id: params.commandId, ok: true, result: params.result ?? null }
|
||||
: { command_id: params.commandId, ok: false, error: params.error })
|
||||
return { accepted: Boolean(result?.accepted) }
|
||||
}
|
||||
case 'message.send':
|
||||
case 'message.stop': {
|
||||
if (this.selected?.storedId !== params.sessionId) throw new Error('Önce bu konuşmayı seçin.')
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "aiturk-ide",
|
||||
"productName": "AITURK IDE",
|
||||
"private": true,
|
||||
"version": "1.0.0-beta.41",
|
||||
"version": "1.0.0-beta.42",
|
||||
"description": "TurkServis resmi yapay zekâ geliştirme ortamı. Hermes Agent tabanlıdır.",
|
||||
"author": "AITURK / TurkServis",
|
||||
"repository": {
|
||||
|
||||
Reference in new issue
Block a user