The AITURK Chrome extension now registers the user's Chrome as the desktop backend's browser controller, the way the ChatGPT extension lets its agent work in the user's browser. The agent's existing browser tools (navigate, snapshot, click, type, scroll, back, press, get_images, vision) run in its own background tabs grouped as "AITURK", or a tab the user hands over. Backend: a local desktop lane in the browser-control broker. One controller registered from the loopback desktop transport is offered to every desktop session as a lane template; each session still gets its own exact scope on first use. The lane has its own switch (browser.extension_control.local_desktop, on by default); with no extension registered, or after it disconnects, the normal browser backend runs unchanged. browser_vision turns the extension's capture into the usual native/auxiliary vision result. Bridge: forwards controller commands only to a connection that registered, refuses unknown or oversized commands, relays results. Extension: service-worker controller with its own native connection, trusted chrome.debugger input (press/release only: a hover move waits 5 s on background tabs), refusal of password and payment-card fields and of browser, extension, file and Web Store pages, panel switch, Stop, tab hand-over, recent actions, Alt+Shift+A and right-click "send to AITURK". Cancelling Chrome's debugging bar stops the agent. Existing panel flows and privacy rules are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
292 lines
11 KiB
Python
292 lines
11 KiB
Python
"""Registry-level browser extension router.
|
|
|
|
This module is the *agent-side* half of the browser-extension-control
|
|
feature: it decides, for one registry ``browser_*`` handler invocation,
|
|
whether the command is executed by an attached extension controller (via
|
|
the :mod:`gateway.browser_control_broker`) or by the existing legacy
|
|
browser backend.
|
|
|
|
Routing contract (exercised by ``tests/tools/test_browser_extension_router.py``):
|
|
|
|
- **Feature off ⇒ legacy, untouched.** When ``enabled`` is false the broker
|
|
is never touched and ``fallback()`` is called exactly once. This is the
|
|
default: ``browser.extension_control.enabled`` is false unless explicitly
|
|
configured, so every real browser action keeps its exact legacy path.
|
|
|
|
- **No server-bound identity ⇒ legacy.** Generic Hermes callers keep the
|
|
existing backend when no authenticated browser-controller identity is bound.
|
|
|
|
- **Bound identity ⇒ authoritative extension lane.** Once the gateway binds a
|
|
browser-controller principal and transport family, missing/ambiguous scope,
|
|
disconnect, or capability mismatch fail closed. A "control this tab" turn
|
|
must never jump to an unrelated local/cloud browser backend.
|
|
|
|
- **Selected controller ⇒ authoritative.** Once a controller is selected the
|
|
command is dispatched to it and its result returned; the legacy backend
|
|
is *never* retried, even when the controller fails (timeout, cancellation,
|
|
rejection, transport error all propagate to the caller).
|
|
|
|
- **Arguments are never mutated.** ``args`` is passed through untouched;
|
|
the broker copies arguments into its command frame itself.
|
|
|
|
The lazy wrapper :func:`routed_browser_handler` is what the ``browser_*``
|
|
registry handlers call. It resolves the feature flag and the process-local
|
|
broker lazily on every invocation so importing this module (or
|
|
``tools.browser_tool``) never pulls in the gateway, and so a mid-process
|
|
config change is honored without restart.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from typing import Any, Callable, Dict, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def lane_enabled(transport_family: Optional[str]) -> bool:
|
|
"""Feature gate for one controller lane.
|
|
|
|
The desktop IDE's own browser lane has its own switch
|
|
(``browser.extension_control.local_desktop``, on by default); every other
|
|
lane keeps the explicit ``browser.extension_control.enabled`` opt-in.
|
|
"""
|
|
from gateway.browser_control_broker import (
|
|
LOCAL_DESKTOP_TRANSPORT_FAMILY,
|
|
browser_control_enabled,
|
|
browser_control_local_enabled,
|
|
)
|
|
|
|
if transport_family == LOCAL_DESKTOP_TRANSPORT_FAMILY:
|
|
return browser_control_local_enabled()
|
|
return browser_control_enabled()
|
|
|
|
|
|
def extension_controller_available(action: str) -> bool:
|
|
"""Whether this request owns one exact controller capable of ``action``.
|
|
|
|
Tool-schema assembly runs inside the API request's session context, before
|
|
a model can call a browser tool. The legacy browser backend's availability
|
|
probe cannot decide whether the extension route is usable, so routeable
|
|
tools consult the process-local broker directly. Missing server-bound
|
|
identity, ambiguous scope, a detached controller, or a capability mismatch
|
|
all fail closed.
|
|
"""
|
|
try:
|
|
from gateway.browser_control_broker import get_browser_control_broker
|
|
from gateway.session_context import get_session_env
|
|
|
|
session_id = get_session_env("HERMES_SESSION_ID", "") or None
|
|
principal_id = get_session_env("HERMES_BROWSER_CONTROL_PRINCIPAL", "") or None
|
|
transport_family = get_session_env(
|
|
"HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY", ""
|
|
) or None
|
|
if not session_id or not principal_id or not transport_family:
|
|
return False
|
|
if not lane_enabled(transport_family):
|
|
return False
|
|
broker = get_browser_control_broker()
|
|
scope = broker.scope_for_session(
|
|
session_id=session_id,
|
|
principal_id=principal_id,
|
|
transport_family=transport_family,
|
|
)
|
|
return scope is not None and broker.select(scope, action) is not None
|
|
except Exception:
|
|
logger.debug(
|
|
"browser extension availability check failed for %s",
|
|
action,
|
|
exc_info=True,
|
|
)
|
|
return False
|
|
|
|
|
|
def route_browser_tool(
|
|
action: str,
|
|
args: Dict[str, Any],
|
|
*,
|
|
fallback: Callable[[], Any],
|
|
broker: Any,
|
|
enabled: bool,
|
|
session_id: Optional[str] = None,
|
|
task_id: Optional[str] = None,
|
|
principal_id: Optional[str] = None,
|
|
transport_family: Optional[str] = None,
|
|
tool_call_id: Optional[str] = "",
|
|
transform: Optional[Callable[[Any], Any]] = None,
|
|
) -> Any:
|
|
"""Route one browser action through the extension-control broker.
|
|
|
|
Parameters
|
|
----------
|
|
action:
|
|
Registry tool name / controller capability, e.g. ``"browser_navigate"``.
|
|
args:
|
|
Tool arguments as received from the model. Never mutated.
|
|
fallback:
|
|
The existing backend handler, called exactly once when the feature is
|
|
off or no server-bound controller identity exists. Must be a
|
|
zero-argument callable.
|
|
broker:
|
|
Object exposing ``scope_for_session(**identity) -> scope|None``,
|
|
``select(scope, capability) -> controller|None`` and
|
|
``dispatch(scope, *, action, arguments, tool_call_id)``. The real
|
|
implementation is ``gateway.browser_control_broker``.
|
|
enabled:
|
|
Feature flag; false bypasses the broker entirely.
|
|
session_id/task_id:
|
|
Caller session hints forwarded to ``scope_for_session``.
|
|
principal_id/transport_family:
|
|
Server-bound caller identity. Both are mandatory when the feature is
|
|
enabled; missing values preserve the existing backend for generic
|
|
Hermes callers.
|
|
tool_call_id:
|
|
Caller tool-call id forwarded verbatim to ``dispatch``.
|
|
transform:
|
|
Optional post-processor for a *controller* result only (never for the
|
|
fallback). Its return value is the tool result as-is, so it may build
|
|
the multimodal envelope (``browser_vision`` turns the controller's
|
|
screenshot into the same result the legacy backend returns).
|
|
|
|
Returns
|
|
-------
|
|
The legacy backend's return value when falling back, or the controller's
|
|
completion result when routed. Exceptions from a selected controller are
|
|
propagated — the legacy backend is never retried after selection.
|
|
"""
|
|
if not enabled:
|
|
return fallback()
|
|
|
|
if not str(principal_id or "").strip() or not str(transport_family or "").strip():
|
|
return fallback()
|
|
|
|
scope = broker.scope_for_session(
|
|
session_id=session_id,
|
|
task_id=task_id,
|
|
principal_id=principal_id,
|
|
transport_family=transport_family,
|
|
)
|
|
if scope is None:
|
|
# A stamped identity alone does not make the extension lane
|
|
# authoritative — authentication happens at transport auth, but the
|
|
# lane only BINDS when a controller actually registers for it. If no
|
|
# controller ever registered, generic callers keep the legacy
|
|
# backend. Once a lane registered (even if the controller is
|
|
# currently offline/ambiguous), fail closed: a "control this tab"
|
|
# session must never silently jump to an unrelated browser.
|
|
lane_bound = getattr(broker, "lane_registered", None)
|
|
if callable(lane_bound) and not lane_bound(
|
|
session_id=session_id,
|
|
task_id=task_id,
|
|
principal_id=principal_id,
|
|
transport_family=transport_family,
|
|
):
|
|
return fallback()
|
|
from gateway.browser_control_broker import ControllerUnavailable
|
|
|
|
raise ControllerUnavailable(
|
|
f"bound browser controller unavailable for {action}"
|
|
)
|
|
|
|
controller = broker.select(scope, action)
|
|
if controller is None:
|
|
from gateway.browser_control_broker import ControllerUnavailable
|
|
|
|
raise ControllerUnavailable(
|
|
f"bound browser controller cannot execute {action}"
|
|
)
|
|
|
|
# A controller was selected: it is authoritative. Never retry through the
|
|
# existing backend, whatever happens here. Registry handlers must return a
|
|
# string (or the dedicated multimodal envelope), while controller transports
|
|
# naturally complete with decoded JSON values. Preserve existing string
|
|
# results byte-for-byte and serialize decoded values at this boundary.
|
|
result = broker.dispatch(
|
|
scope, action=action, arguments=args, tool_call_id=tool_call_id
|
|
)
|
|
if transform is not None:
|
|
return transform(result)
|
|
if isinstance(result, str):
|
|
return result
|
|
return json.dumps(result, ensure_ascii=False)
|
|
|
|
|
|
def current_tool_call_id() -> str:
|
|
"""Return the active tool_call_id, or ``""`` when none is bound.
|
|
|
|
The agent executor binds the id via
|
|
``tools.approval.set_current_observability_context`` immediately before
|
|
registry dispatch, so the registry handler (and this router) can read it
|
|
back from the same context. Bare/offline callers have no binding.
|
|
"""
|
|
try:
|
|
from tools.approval import _approval_tool_call_id
|
|
|
|
return _approval_tool_call_id.get() or ""
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def routed_browser_handler(
|
|
action: str,
|
|
args: Dict[str, Any],
|
|
*,
|
|
fallback: Callable[[], Any],
|
|
task_id: Optional[str] = None,
|
|
session_id: Optional[str] = None,
|
|
principal_id: Optional[str] = None,
|
|
transport_family: Optional[str] = None,
|
|
tool_call_id: Optional[str] = None,
|
|
transform: Optional[Callable[[Any], Any]] = None,
|
|
) -> Any:
|
|
"""Lazy registry-handler route wrapper for ``browser_*`` tools.
|
|
|
|
Resolves the feature flag and process-local broker lazily so the
|
|
default (feature off) path costs one cached config read and an immediate
|
|
fallback, and so importing ``tools.browser_tool`` never imports the
|
|
gateway. When the gateway cannot be imported or the feature is off, the
|
|
legacy handler runs unchanged.
|
|
"""
|
|
try:
|
|
from gateway.browser_control_broker import get_browser_control_broker
|
|
except Exception as exc: # pragma: no cover - defensive, gateway always present
|
|
logger.debug(
|
|
"browser extension router unavailable (%s); using legacy backend",
|
|
exc,
|
|
)
|
|
return fallback()
|
|
|
|
try:
|
|
from gateway.session_context import get_session_env
|
|
|
|
session_id = session_id or get_session_env("HERMES_SESSION_ID", "") or None
|
|
principal_id = principal_id or get_session_env(
|
|
"HERMES_BROWSER_CONTROL_PRINCIPAL", ""
|
|
) or None
|
|
transport_family = transport_family or get_session_env(
|
|
"HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY", ""
|
|
) or None
|
|
except Exception:
|
|
pass
|
|
|
|
if not lane_enabled(transport_family):
|
|
return fallback()
|
|
|
|
if tool_call_id is None:
|
|
tool_call_id = current_tool_call_id()
|
|
|
|
return route_browser_tool(
|
|
action,
|
|
args,
|
|
fallback=fallback,
|
|
broker=get_browser_control_broker(),
|
|
enabled=True,
|
|
session_id=session_id,
|
|
task_id=task_id,
|
|
principal_id=principal_id,
|
|
transport_family=transport_family,
|
|
tool_call_id=tool_call_id,
|
|
transform=transform,
|
|
)
|