yilsem/aiturk-hermes-ide
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

feat(chrome): let the IDE agent use the user's Chrome and beta.42
Server CI and Windows package / verify (push) Successful in 3m33s
Details
Server CI and Windows package / windows-package (push) Successful in 7m36s
Details

Browse Source 
The AITURK Chrome extension now registers the user's Chrome as the desktop
backend's browser controller, the way the ChatGPT extension lets its agent
work in the user's browser. The agent's existing browser tools (navigate,
snapshot, click, type, scroll, back, press, get_images, vision) run in its
own background tabs grouped as "AITURK", or a tab the user hands over.

Backend: a local desktop lane in the browser-control broker. One controller
registered from the loopback desktop transport is offered to every desktop
session as a lane template; each session still gets its own exact scope on
first use. The lane has its own switch (browser.extension_control.local_desktop,
on by default); with no extension registered, or after it disconnects, the
normal browser backend runs unchanged. browser_vision turns the extension's
capture into the usual native/auxiliary vision result.

Bridge: forwards controller commands only to a connection that registered,
refuses unknown or oversized commands, relays results.

Extension: service-worker controller with its own native connection,
trusted chrome.debugger input (press/release only: a hover move waits 5 s on
background tabs), refusal of password and payment-card fields and of
browser, extension, file and Web Store pages, panel switch, Stop, tab
hand-over, recent actions, Alt+Shift+A and right-click "send to AITURK".
Cancelling Chrome's debugging bar stops the agent. Existing panel flows and
privacy rules are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
yilsemandClaude Opus 5.5 committed 2026-10-15 08:38:52 +03:00
1 parent 5c76c24c9c
commit 86a6621936
32 files changed
+2755 -195
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/browser_tool.py
+251 -146
View File
@@ -5676,152 +5676,14 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
),
}, ensure_ascii=False)
# NOTE: the full-resolution base64 encode is deliberately deferred.
# The native fast path below sizes its own history-reuse embed via
# _resize_image_for_vision (stat-based quick estimate — no full-res
# encode when oversized), and only the aux-LLM fallback path needs
# the one-shot full-res data URL.
# Fast path: when native image routing is in effect for the active main
# model, attach the screenshot directly instead of describing it through
# an auxiliary vision LLM. The model inspects the pixels on its next
# turn — no aux call, no information loss. Consistent with vision_analyze.
from tools.vision_tools import (
_EMBED_MAX_DIMENSION,
_EMBED_TARGET_BYTES,
_build_native_vision_tool_result,
_resize_image_for_vision,
_should_use_native_vision_fast_path,
return _analyze_browser_screenshot(
screenshot_path,
question,
annotate,
result,
fallback_warning=_lp_fallback_warning,
)
if _should_use_native_vision_fast_path():
# History-reuse cap (#92699): this embed is baked into the tool
# result and re-sent on every later turn, exactly like
# vision_analyze's native path — apply the same proactive resize
# so full-res screenshots can't enter immutable history uncapped.
# The helper's internal stat/dimension quick-estimate skips the
# resize (and encodes directly) when the screenshot is already
# under both caps, so no full-res base64 is built just to be
# thrown away. Fail-open: without Pillow it falls back to the
# raw bytes and the compressor's keep-newest pass still retires
# stale embeds.
data_url = _resize_image_for_vision(
screenshot_path,
mime_type="image/png",
max_base64_bytes=_EMBED_TARGET_BYTES,
max_dimension=_EMBED_MAX_DIMENSION,
force_jpeg=True,
)
native_result = _build_native_vision_tool_result(
image_url=str(screenshot_path),
question=question,
image_data_url=data_url,
image_size_bytes=screenshot_path.stat().st_size,
)
meta = native_result.setdefault("meta", {})
meta["screenshot_path"] = str(screenshot_path)
if _lp_fallback_warning:
meta["fallback_warning"] = _lp_fallback_warning
if annotate and result.get("data", {}).get("annotations"):
meta["annotations"] = result["data"]["annotations"]
native_result["text_summary"] = (
f"{native_result.get('text_summary', '')} "
f"Screenshot path: {screenshot_path}"
).strip()
return native_result
vision_prompt = (
f"You are analyzing a screenshot of a web browser.\n\n"
f"User's question: {question}\n\n"
f"Provide a detailed and helpful answer based on what you see in the screenshot. "
f"If there are interactive elements, describe them. If there are verification challenges "
f"or CAPTCHAs, describe what type they are and what action might be needed. "
f"Focus on answering the user's specific question."
)
# Aux-LLM path: one-shot analysis, not baked into history — encode at
# full resolution here (the pre-existing 5 MB oversize guard below
# still applies).
_screenshot_bytes = screenshot_path.read_bytes()
_screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii")
data_url = f"data:image/png;base64,{_screenshot_b64}"
# Use the centralized LLM router
vision_model = _get_vision_model()
logger.debug("browser_vision: analysing screenshot (%d bytes)",
len(_screenshot_bytes))
# Read vision timeout/temperature from config (auxiliary.vision.*).
# Local vision models (llama.cpp, ollama) can take well over 30s for
# screenshot analysis, so the default timeout must be generous.
vision_timeout = 120.0
vision_temperature = 0.1
try:
from hermes_cli.config import load_config
_cfg = load_config()
_vision_cfg = cfg_get(_cfg, "auxiliary", "vision", default={})
_vt = _vision_cfg.get("timeout")
if _vt is not None:
vision_timeout = float(_vt)
_vtemp = _vision_cfg.get("temperature")
if _vtemp is not None:
vision_temperature = float(_vtemp)
except Exception:
pass
call_kwargs = {
"task": "vision",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": vision_prompt},
{"type": "image_url", "image_url": {"url": data_url}},
],
}
],
"temperature": vision_temperature,
"timeout": vision_timeout,
}
if vision_model:
call_kwargs["model"] = vision_model
# Try full-size screenshot; on size-related rejection, downscale and retry.
try:
response = _lazy_call_llm(**call_kwargs)
except Exception as _api_err:
from tools.vision_tools import (
_is_image_size_error, _resize_image_for_vision, _RESIZE_TARGET_BYTES,
)
if (_is_image_size_error(_api_err)
and len(data_url) > _RESIZE_TARGET_BYTES):
logger.info(
"Vision API rejected screenshot (%.1f MB); "
"auto-resizing to ~%.0f MB and retrying...",
len(data_url) / (1024 * 1024),
_RESIZE_TARGET_BYTES / (1024 * 1024),
)
data_url = _resize_image_for_vision(
screenshot_path, mime_type="image/png")
call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url
response = _lazy_call_llm(**call_kwargs)
else:
raise
analysis = (response.choices[0].message.content or "").strip()
# Redact secrets the vision LLM may have read from the screenshot.
from agent.redact import redact_sensitive_text
analysis = redact_sensitive_text(analysis)
response_data = {
"success": True,
"analysis": analysis or "Vision analysis returned no content.",
"screenshot_path": str(screenshot_path),
}
_copy_fallback_warning(response_data, result)
# Include annotation data if annotated screenshot was taken
if annotate and result.get("data", {}).get("annotations"):
response_data["annotations"] = result["data"]["annotations"]
return json.dumps(response_data, ensure_ascii=False)
except Exception as e:
# Keep the screenshot if it was captured successfully — the failure is
# in the LLM vision analysis, not the capture. Deleting a valid
@@ -5836,6 +5698,243 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
return json.dumps(error_info, ensure_ascii=False)
#: Largest decoded screenshot accepted from a browser-extension controller.
_CONTROLLER_SCREENSHOT_MAX_BYTES = 8 * 1024 * 1024
def _browser_vision_from_controller(
controller_result: Any, question: str, annotate: bool = False
) -> Union[str, Dict[str, Any]]:
"""Turn an extension controller's screenshot into a browser_vision result.
The controller (the user's own browser) only captures the visible tab:
``{"success": true, "screenshot": <base64>, "mime": "image/jpeg", ...}``.
Saving the image and the native/auxiliary vision analysis stay here, so
the model receives exactly what the legacy backend would return.
"""
import base64
import uuid as uuid_mod
from hermes_constants import get_hermes_dir
data = controller_result
if isinstance(data, str):
try:
data = json.loads(data)
except (TypeError, ValueError):
return data
if not isinstance(data, dict):
return json.dumps(
{"success": False, "error": "Browser returned no screenshot."},
ensure_ascii=False,
)
if data.get("success") is not True or not isinstance(data.get("screenshot"), str):
return json.dumps(
{
"success": False,
"error": str(data.get("error") or "Browser returned no screenshot."),
},
ensure_ascii=False,
)
mime = "image/png" if data.get("mime") == "image/png" else "image/jpeg"
try:
raw = base64.b64decode(data["screenshot"], validate=True)
except (ValueError, TypeError):
return json.dumps(
{"success": False, "error": "Browser returned an unreadable screenshot."},
ensure_ascii=False,
)
if not raw or len(raw) > _CONTROLLER_SCREENSHOT_MAX_BYTES:
return json.dumps(
{"success": False, "error": "Browser screenshot is empty or too large."},
ensure_ascii=False,
)
screenshots_dir = get_hermes_dir("cache/screenshots", "browser_screenshots")
screenshots_dir.mkdir(parents=True, exist_ok=True)
_cleanup_old_screenshots(screenshots_dir, max_age_hours=24)
suffix = ".png" if mime == "image/png" else ".jpg"
screenshot_path = screenshots_dir / f"browser_screenshot_{uuid_mod.uuid4().hex}{suffix}"
screenshot_path.write_bytes(raw)
capture = {"success": True, "data": {"path": str(screenshot_path)}}
annotations = data.get("annotations")
if annotate and isinstance(annotations, list):
capture["data"]["annotations"] = annotations
try:
return _analyze_browser_screenshot(
screenshot_path, question, annotate, capture, mime_type=mime
)
except Exception as e:
logger.warning("browser_vision (extension) failed: %s", e, exc_info=True)
return json.dumps(
{
"success": False,
"error": f"Error during vision analysis: {str(e)}",
"screenshot_path": str(screenshot_path),
"note": "Screenshot was captured but vision analysis failed. You can still share it via MEDIA:<path>.",
},
ensure_ascii=False,
)
def _analyze_browser_screenshot(
screenshot_path: Path,
question: str,
annotate: bool,
result: Dict[str, Any],
*,
mime_type: str = "image/png",
fallback_warning: Optional[str] = None,
) -> Union[str, Dict[str, Any]]:
"""Native-attach or auxiliary-LLM analysis of one saved browser screenshot."""
import base64
_lp_fallback_warning = fallback_warning
# NOTE: the full-resolution base64 encode is deliberately deferred.
# The native fast path below sizes its own history-reuse embed via
# _resize_image_for_vision (stat-based quick estimate — no full-res
# encode when oversized), and only the aux-LLM fallback path needs
# the one-shot full-res data URL.
# Fast path: when native image routing is in effect for the active main
# model, attach the screenshot directly instead of describing it through
# an auxiliary vision LLM. The model inspects the pixels on its next
# turn — no aux call, no information loss. Consistent with vision_analyze.
from tools.vision_tools import (
_EMBED_MAX_DIMENSION,
_EMBED_TARGET_BYTES,
_build_native_vision_tool_result,
_resize_image_for_vision,
_should_use_native_vision_fast_path,
)
if _should_use_native_vision_fast_path():
# History-reuse cap (#92699): this embed is baked into the tool
# result and re-sent on every later turn, exactly like
# vision_analyze's native path — apply the same proactive resize
# so full-res screenshots can't enter immutable history uncapped.
# The helper's internal stat/dimension quick-estimate skips the
# resize (and encodes directly) when the screenshot is already
# under both caps, so no full-res base64 is built just to be
# thrown away. Fail-open: without Pillow it falls back to the
# raw bytes and the compressor's keep-newest pass still retires
# stale embeds.
data_url = _resize_image_for_vision(
screenshot_path,
mime_type=mime_type,
max_base64_bytes=_EMBED_TARGET_BYTES,
max_dimension=_EMBED_MAX_DIMENSION,
force_jpeg=True,
)
native_result = _build_native_vision_tool_result(
image_url=str(screenshot_path),
question=question,
image_data_url=data_url,
image_size_bytes=screenshot_path.stat().st_size,
)
meta = native_result.setdefault("meta", {})
meta["screenshot_path"] = str(screenshot_path)
if _lp_fallback_warning:
meta["fallback_warning"] = _lp_fallback_warning
if annotate and result.get("data", {}).get("annotations"):
meta["annotations"] = result["data"]["annotations"]
native_result["text_summary"] = (
f"{native_result.get('text_summary', '')} "
f"Screenshot path: {screenshot_path}"
).strip()
return native_result
vision_prompt = (
f"You are analyzing a screenshot of a web browser.\n\n"
f"User's question: {question}\n\n"
f"Provide a detailed and helpful answer based on what you see in the screenshot. "
f"If there are interactive elements, describe them. If there are verification challenges "
f"or CAPTCHAs, describe what type they are and what action might be needed. "
f"Focus on answering the user's specific question."
)
# Aux-LLM path: one-shot analysis, not baked into history — encode at
# full resolution here (the pre-existing 5 MB oversize guard below
# still applies).
_screenshot_bytes = screenshot_path.read_bytes()
_screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii")
data_url = f"data:{mime_type};base64,{_screenshot_b64}"
# Use the centralized LLM router
vision_model = _get_vision_model()
logger.debug("browser_vision: analysing screenshot (%d bytes)",
len(_screenshot_bytes))
# Read vision timeout/temperature from config (auxiliary.vision.*).
# Local vision models (llama.cpp, ollama) can take well over 30s for
# screenshot analysis, so the default timeout must be generous.
vision_timeout = 120.0
vision_temperature = 0.1
try:
from hermes_cli.config import load_config
_cfg = load_config()
_vision_cfg = cfg_get(_cfg, "auxiliary", "vision", default={})
_vt = _vision_cfg.get("timeout")
if _vt is not None:
vision_timeout = float(_vt)
_vtemp = _vision_cfg.get("temperature")
if _vtemp is not None:
vision_temperature = float(_vtemp)
except Exception:
pass
call_kwargs = {
"task": "vision",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": vision_prompt},
{"type": "image_url", "image_url": {"url": data_url}},
],
}
],
"temperature": vision_temperature,
"timeout": vision_timeout,
}
if vision_model:
call_kwargs["model"] = vision_model
# Try full-size screenshot; on size-related rejection, downscale and retry.
try:
response = _lazy_call_llm(**call_kwargs)
except Exception as _api_err:
from tools.vision_tools import (
_is_image_size_error, _resize_image_for_vision, _RESIZE_TARGET_BYTES,
)
if (_is_image_size_error(_api_err)
and len(data_url) > _RESIZE_TARGET_BYTES):
logger.info(
"Vision API rejected screenshot (%.1f MB); "
"auto-resizing to ~%.0f MB and retrying...",
len(data_url) / (1024 * 1024),
_RESIZE_TARGET_BYTES / (1024 * 1024),
)
data_url = _resize_image_for_vision(
screenshot_path, mime_type=mime_type)
call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url
response = _lazy_call_llm(**call_kwargs)
else:
raise
analysis = (response.choices[0].message.content or "").strip()
# Redact secrets the vision LLM may have read from the screenshot.
from agent.redact import redact_sensitive_text
analysis = redact_sensitive_text(analysis)
response_data = {
"success": True,
"analysis": analysis or "Vision analysis returned no content.",
"screenshot_path": str(screenshot_path),
}
_copy_fallback_warning(response_data, result)
# Include annotation data if annotated screenshot was taken
if annotate and result.get("data", {}).get("annotations"):
response_data["annotations"] = result["data"]["annotations"]
return json.dumps(response_data, ensure_ascii=False)
def _cleanup_old_screenshots(screenshots_dir, max_age_hours=24):
"""Remove browser screenshots older than max_age_hours to prevent disk bloat.
@@ -6381,7 +6480,10 @@ def check_browser_vision_requirements() -> bool:
fails at call time with a cryptic provider-side error like
``unknown variant `image_url`, expected `text``` (issue #31179).
"""
if not check_browser_requirements():
if not (
check_browser_requirements()
or extension_controller_available("browser_vision")
):
return False
try:
from tools.vision_tools import check_vision_requirements
@@ -6602,7 +6704,7 @@ registry.register(
fallback=lambda: browser_get_images(task_id=kw.get("task_id")),
**_browser_router_kw(kw),
),
check_fn=check_browser_requirements,
check_fn=lambda: check_browser_routed_requirements("browser_get_images"),
emoji="🖼️",
)
registry.register(
@@ -6613,6 +6715,9 @@ registry.register(
"browser_vision",
args,
fallback=lambda: browser_vision(question=args.get("question", ""), annotate=args.get("annotate", False), task_id=kw.get("task_id")),
transform=lambda result: _browser_vision_from_controller(
result, args.get("question", ""), bool(args.get("annotate", False))
),
**_browser_router_kw(kw),
),
check_fn=check_browser_vision_requirements,