feat(chrome): let the IDE agent use the user's Chrome and beta.42

The AITURK Chrome extension now registers the user's Chrome as the desktop
backend's browser controller, the way the ChatGPT extension lets its agent
work in the user's browser. The agent's existing browser tools (navigate,
snapshot, click, type, scroll, back, press, get_images, vision) run in its
own background tabs grouped as "AITURK", or a tab the user hands over.

Backend: a local desktop lane in the browser-control broker. One controller
registered from the loopback desktop transport is offered to every desktop
session as a lane template; each session still gets its own exact scope on
first use. The lane has its own switch (browser.extension_control.local_desktop,
on by default); with no extension registered, or after it disconnects, the
normal browser backend runs unchanged. browser_vision turns the extension's
capture into the usual native/auxiliary vision result.

Bridge: forwards controller commands only to a connection that registered,
refuses unknown or oversized commands, relays results.

Extension: service-worker controller with its own native connection,
trusted chrome.debugger input (press/release only: a hover move waits 5 s on
background tabs), refusal of password and payment-card fields and of
browser, extension, file and Web Store pages, panel switch, Stop, tab
hand-over, recent actions, Alt+Shift+A and right-click "send to AITURK".
Cancelling Chrome's debugging bar stops the agent. Existing panel flows and
privacy rules are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

gateway/browser_control_broker.py

@@ -69,7 +69,7 @@ import logging
 import secrets
 import threading
 import time
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from typing import Any, Callable, Dict, Optional
 
 logger = logging.getLogger(__name__)
@@ -94,6 +94,7 @@ BROWSER_CONTROL_CAPABILITIES = frozenset(
         "controller.noop",
         "browser_back",
         "browser_click",
+        "browser_get_images",
         "browser_navigate",
         "browser_press",
         "browser_screenshot",
@@ -102,9 +103,22 @@ BROWSER_CONTROL_CAPABILITIES = frozenset(
         "browser_tab_activate",
         "browser_tabs",
         "browser_type",
+        # The controller only captures the visible tab; the agent side saves
+        # the image and runs the usual native/auxiliary vision analysis.
+        "browser_vision",
     }
 )
 
+#: Transport family for the desktop IDE's own browser (the AITURK Chrome
+#: extension behind the desktop's native-messaging bridge). Its controller is
+#: registered once per backend as a lane template and serves every desktop
+#: session of that backend; see :meth:`BrowserControlBroker.attach_lane_template`.
+LOCAL_DESKTOP_TRANSPORT_FAMILY = "desktop-local-ws"
+#: Principal of the single local desktop user. The desktop backend is
+#: loopback-only and its session token already grants every desktop session,
+#: so there is exactly one local principal per backend process.
+LOCAL_DESKTOP_PRINCIPAL = "principal:desktop:local"
+
 #: Privileged capabilities (Phase 8 Task 30) that are never negotiable through
 #: the base allowlist. ``browser_evaluate`` executes JavaScript in the page
 #: context; ``browser_cdp`` is raw CDP. Both are fail-closed unless the broker
@@ -304,6 +318,23 @@ class _PendingCommand:
     result: Any = None
 
 
+@dataclass
+class _LaneTemplate:
+    """One controller offered to every session of one authenticated lane.
+
+    A template never matches a session by itself. The first lookup for a
+    session materializes an exact per-session scope through :meth:`attach`,
+    so selection, dispatch, cancellation and completion keep the broker's
+    exact-identity contract. ``send_for(session_id)`` returns the transport
+    writer that addresses that session's frames.
+    """
+
+    template: ControllerScope
+    send_for: Callable[[str], Callable[[dict], None]]
+    owner: Any
+    derived: Dict[str, ControllerScope] = field(default_factory=dict)
+
+
 class BrowserControlBroker:
     """Thread-safe broker core binding controllers to callers.
 
@@ -347,6 +378,10 @@ class BrowserControlBroker:
         # listener attaches one store per profile so profile A touching the
         # artifact route first can never pin profile B to A's physical root.
         self._artifact_stores: Dict[Optional[str], Any] = {}
+        # Lane templates keyed by (principal_id, transport_family). Only the
+        # local desktop lane uses them; every other transport registers one
+        # exact scope per session.
+        self._lane_templates: Dict[tuple, _LaneTemplate] = {}
 
     def _developer_mode_now(self) -> bool:
         """Current Developer Mode authority (live config unless pinned)."""
@@ -575,6 +610,163 @@ class BrowserControlBroker:
             ]
         return len(matches) == 1
 
+    # ------------------------------------------------------------------
+    # Lane templates (one controller for every session of a lane)
+    # ------------------------------------------------------------------
+
+    def attach_lane_template(
+        self,
+        template: ControllerScope,
+        send_for: Callable[[str], Callable[[dict], None]],
+        *,
+        owner: Any,
+    ) -> None:
+        """Offer one controller to every session of ``template``'s lane.
+
+        The lane is ``(principal_id, transport_family)``; ``template`` carries
+        no session id. Re-registering the same controller (same profile,
+        controller id and browser profile) refreshes the transport and
+        negotiated capabilities of the sessions it already serves without
+        cancelling their work. A different controller hard-replaces the old
+        one: its per-session scopes are detached first, cancelling their
+        pending commands, exactly like a replacement in :meth:`attach`.
+        """
+        template = replace(template, session_id=None)
+        key = (template.principal_id, template.transport_family)
+        lane = _LaneTemplate(template=template, send_for=send_for, owner=owner)
+        with self._lock:
+            previous = self._lane_templates.get(key)
+            self._lane_templates[key] = lane
+        if previous is None:
+            return
+        same_controller = (
+            previous.template.profile_id == template.profile_id
+            and previous.template.controller_id == template.controller_id
+            and previous.template.browser_profile_id == template.browser_profile_id
+        )
+        for session_id, scope in list(previous.derived.items()):
+            if not same_controller:
+                self._detach_identity(scope, notify_controller=False)
+                continue
+            refreshed = replace(template, session_id=session_id)
+            try:
+                self.attach(refreshed, send_for(session_id), owner=owner)
+            except Exception:
+                logger.exception("failed to refresh browser-controller lane scope")
+                self._detach_identity(scope, notify_controller=False)
+                continue
+            with self._lock:
+                if self._lane_templates.get(key) is lane:
+                    lane.derived[session_id] = refreshed
+
+    def detach_lane_template(
+        self,
+        *,
+        principal_id: Optional[str],
+        transport_family: Optional[str],
+        owner: Any = _OWNER_UNSET,
+        notify_controller: bool = True,
+    ) -> bool:
+        """Remove a lane template and hard-detach every session it served."""
+        key = (principal_id, transport_family)
+        with self._lock:
+            lane = self._lane_templates.get(key)
+            if lane is None:
+                return False
+            if owner is not _OWNER_UNSET and lane.owner is not owner:
+                return False
+            del self._lane_templates[key]
+            derived = list(lane.derived.values())
+        for scope in derived:
+            self._detach_identity(scope, notify_controller=notify_controller)
+        return True
+
+    def lane_template(
+        self, *, principal_id: Optional[str], transport_family: Optional[str]
+    ) -> Optional[ControllerScope]:
+        """Return the template scope offered to a lane, if any."""
+        with self._lock:
+            lane = self._lane_templates.get((principal_id, transport_family))
+            return lane.template if lane is not None else None
+
+    def _materialize_lane_scope(
+        self, target: str, principal: str, family: str
+    ) -> Optional[ControllerScope]:
+        """Attach the exact per-session scope a lane template offers."""
+        key = (principal, family)
+        with self._lock:
+            lane = self._lane_templates.get(key)
+            if lane is None:
+                return None
+            existing = lane.derived.get(target)
+            if existing is not None and any(
+                _same_scope_identity(candidate, existing)
+                for candidate in self._controllers
+            ):
+                return existing
+            scope = replace(lane.template, session_id=target)
+            send_for, owner = lane.send_for, lane.owner
+        self.attach(scope, send_for(target), owner=owner)
+        with self._lock:
+            if self._lane_templates.get(key) is lane:
+                lane.derived[target] = scope
+                return scope
+        # The template was replaced or removed while attaching: never leave a
+        # scope behind that its lane no longer vouches for.
+        self._detach_identity(scope, notify_controller=False)
+        return None
+
+    def _detach_identity(
+        self, scope: ControllerScope, *, notify_controller: bool = True
+    ) -> None:
+        """Detach whatever scope currently carries ``scope``'s identity.
+
+        A refresh may have re-keyed the scope with new capabilities, so an
+        exact-key lookup could miss it.
+        """
+        with self._lock:
+            current = next(
+                (
+                    candidate
+                    for candidate in self._controllers
+                    if _same_scope_identity(candidate, scope)
+                ),
+                None,
+            )
+        if current is not None:
+            self.detach(current, notify_controller=notify_controller)
+
+    def complete_for_owner(
+        self,
+        command_id: str,
+        *,
+        owner: Any,
+        ok: bool,
+        result: Any = None,
+    ) -> bool:
+        """Complete a command only when ``owner`` drives its controller.
+
+        Lane controllers serve many sessions over one transport, so their
+        results are admitted by transport ownership instead of a session
+        lookup. Unknown, foreign or already-resolved ids report ``False``.
+        """
+        with self._lock:
+            pending = self._pending.get(command_id)
+            if pending is None or pending.done:
+                return False
+            controller = next(
+                (
+                    candidate
+                    for candidate in self._controllers.values()
+                    if _same_scope_identity(candidate.scope, pending.scope)
+                ),
+                None,
+            )
+            if controller is None or controller.owner is not owner:
+                return False
+            scope = pending.scope
+        return self.complete(command_id, scope=scope, ok=ok, result=result)
+
     def disconnect(
         self,
         scope: ControllerScope,
@@ -951,7 +1143,12 @@ class BrowserControlBroker:
                 and scope.principal_id == principal
                 and scope.transport_family == family
             ]
-        return matches[0] if len(matches) == 1 else None
+            has_template = (principal, family) in self._lane_templates
+        if matches:
+            return matches[0] if len(matches) == 1 else None
+        if has_template:
+            return self._materialize_lane_scope(target, principal, family)
+        return None
 
     def lane_registered(
         self,
@@ -975,6 +1172,8 @@ class BrowserControlBroker:
         if not target or not principal or not family:
             return False
         with self._lock:
+            if (principal, family) in self._lane_templates:
+                return True
             return any(
                 scope.session_id == target
                 and scope.principal_id == principal
@@ -982,8 +1181,30 @@ class BrowserControlBroker:
                 for scope in self._controllers
             )
 
+    def _detach_lane_templates_owned_by(self, owner: Any) -> int:
+        with self._lock:
+            keys = [
+                key
+                for key, lane in self._lane_templates.items()
+                if lane.owner is owner
+            ]
+        for principal_id, transport_family in keys:
+            self.detach_lane_template(
+                principal_id=principal_id,
+                transport_family=transport_family,
+                owner=owner,
+            )
+        return len(keys)
+
     def disconnect_owner(self, owner: Any) -> int:
-        """Mark every controller owned by one lost transport offline."""
+        """Mark every controller owned by one lost transport offline.
+
+        Lane templates are the exception: a lane controller that goes away
+        (the user closed Chrome, or the IDE bridge reconnected) is removed
+        outright so its sessions go back to their normal browser instead of
+        failing closed until some later reconnect.
+        """
+        self._detach_lane_templates_owned_by(owner)
         with self._lock:
             scopes = [
                 scope
@@ -997,6 +1218,7 @@ class BrowserControlBroker:
 
     def detach_owner(self, owner: Any, *, notify_controller: bool = True) -> int:
         """Hard-detach every controller owned by one transport connection."""
+        self._detach_lane_templates_owned_by(owner)
         with self._lock:
             scopes = [
                 scope
@@ -1014,6 +1236,7 @@ class BrowserControlBroker:
     def reset(self) -> None:
         """Fail all live work closed and clear tickets (tests/shutdown)."""
         with self._lock:
+            self._lane_templates.clear()
             scopes = list(self._controllers)
         for scope in scopes:
             self.detach(scope)
@@ -1065,3 +1288,30 @@ def browser_control_enabled(config: Optional[dict] = None) -> bool:
     if not isinstance(extension_control, dict):
         return False
     return extension_control.get("enabled", False) is True
+
+
+def browser_control_local_enabled(config: Optional[dict] = None) -> bool:
+    """Whether the desktop IDE's own browser may become a lane controller.
+
+    Separate from :func:`browser_control_enabled` because the local lane is a
+    different trust boundary: the controller is the user's own browser on the
+    loopback desktop backend, so it is on unless the user switched it off
+    (``browser.extension_control.local_desktop: false``). Registering the
+    controller is still an explicit act of the browser extension.
+    """
+    if config is None:
+        try:
+            from hermes_cli.config import load_config_readonly
+
+            config = load_config_readonly()
+        except Exception:
+            return True
+    if not isinstance(config, dict):
+        return True
+    browser = config.get("browser")
+    if not isinstance(browser, dict):
+        return True
+    extension_control = browser.get("extension_control")
+    if not isinstance(extension_control, dict):
+        return True
+    return extension_control.get("local_desktop", True) is not False