diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index d656935..8e2fbc8 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -32,6 +32,8 @@ jobs: "$RUNNER_TEMP/ci-venv/bin/python" -m pytest tests/tools/test_protected_instruction_unattended.py tests/tools/test_file_write_safety.py tests/tools/test_computer_use_approval_isolation.py tests/cli/test_cli_approval_ui.py tests/cli/test_single_query_session_finalize.py -q -p no:cacheprovider -rf - name: Codex bridge regression tests run: '"$RUNNER_TEMP/ci-venv/bin/python" -m unittest discover -s apps/desktop/assets/codex-bridge' + - name: Project scoped and bounded search regressions + run: '"$RUNNER_TEMP/ci-venv/bin/python" -m pytest tests/tools/test_search_project_scope.py tests/tools/test_search_files_cpu_windows.py tests/tools/test_search_zero_match_and_multipath.py tests/tools/test_search_giant_line_containment.py tests/tools/test_search_budget_truncation.py -q -p no:cacheprovider -rf' - name: Native watch and speech mux pytest regressions run: '"$RUNNER_TEMP/ci-venv/bin/python" -m pytest apps/desktop/assets/codex-bridge/test_bekci_watch.py apps/desktop/assets/codex-bridge/test_native_speech_mux.py apps/desktop/assets/codex-bridge/test_live_limits.py -q -p no:cacheprovider -rf' - name: Mobile and desktop startup regression tests diff --git a/agent/search_policy.py b/agent/search_policy.py index 2972d5c..be4ada6 100644 --- a/agent/search_policy.py +++ b/agent/search_policy.py @@ -8,6 +8,66 @@ path; broad diagnostic probes should apply this policy to recursive walks. from __future__ import annotations +from pathlib import Path + +SEARCH_RG_THREADS = 2 +SEARCH_MAX_CONCURRENT = 2 + +_PROJECT_MARKERS = ('.git', '.hg', 'pyproject.toml', 'package.json', + 'Cargo.toml', 'go.mod', 'CMakeLists.txt') + + +def _project_root(path): + """Prefer the enclosing repository over nested monorepo packages.""" + package = None + for parent in (path, *path.parents): + if (parent / '.git').exists() or (parent / '.hg').exists(): + return parent + if package is None and any((parent / marker).exists() for marker in _PROJECT_MARKERS): + package = parent + return package + + +def local_search_scope(path, cwd, home): + """Resolve broad local roots without starting a recursive search process. + + Returns (root, notice, error). Only one level of candidate directories is + inspected; no repository-wide discovery is needed to reject an umbrella. + Real paths prevent symlink aliases from bypassing broad-root detection. + """ + root = Path(path) + if not root.is_absolute(): + root = Path(cwd) / root + root = root.resolve() + if not root.is_dir(): + return path, None, None + home = Path(home).resolve() + project = _project_root(Path(cwd).resolve()) + root_project = _project_root(root) + if root_project is not None: + return path, None, None + candidates = [] + try: + # Bound metadata work even in directories containing many entries. + from itertools import islice + for child in islice(root.iterdir(), 256): + if child.name in SEARCH_PRUNE_DIR_NAMES or not child.is_dir(): + continue + if any((child / marker).exists() for marker in _PROJECT_MARKERS): + candidates.append(str(child)) + except OSError: + pass + broad = (root == Path(root.anchor) or root == home or root in home.parents + or len(candidates) > 1 or (project is not None and root in project.parents)) + if not broad: + return path, None, None + if project is not None and root in project.parents: + chosen = str(project) + return chosen, f'Search scoped to current project: {chosen}. Set allow_broad=true for the requested umbrella.', None + choices = ', '.join(sorted(candidates)[:12]) or 'choose an explicit project directory' + return path, None, ('Search scope needed: requested root spans projects or a home/drive. ' + f'Candidates: {choices}. Set allow_broad=true only for an intentional broad search.') + # Keep this policy conservative and name-based so it works for local and remote # shell backends alike. The same set is used by context discovery and search diff --git a/apps/desktop/assets/codex-bridge/bridge.py b/apps/desktop/assets/codex-bridge/bridge.py index b48ac25..82ca46b 100644 --- a/apps/desktop/assets/codex-bridge/bridge.py +++ b/apps/desktop/assets/codex-bridge/bridge.py @@ -34,7 +34,7 @@ import urllib.request import gecis_context HERE = os.path.dirname(os.path.abspath(__file__)) -SURUM = "16" # AITURK IDE'nin bekledigi kopru surumu (codex-link-config.ts CODEX_BRIDGE_VERSION) +SURUM = "17" # AITURK IDE'nin bekledigi kopru surumu (codex-link-config.ts CODEX_BRIDGE_VERSION) try: with open(os.path.abspath(__file__), "rb") as _f: KOD = hashlib.sha256(_f.read()).hexdigest()[:8] @@ -484,15 +484,21 @@ class H(http.server.BaseHTTPRequestHandler): source = TRANSFERS.get(scope) if thread else None # A completed return to the subscription ends migration. Historical # foreign IDs must not keep later native turns in the preparer. - mixed = ek_model_mu(source or "") or (source is None and gecis_context.foreign(req)) - if external or mixed: + mixed = ek_model_mu(source or "") or (source is None and gecis_context.foreign_history(req)) + # Format repair is stateless: the saved transcript can still contain + # foreign wrappers after a successful native return (or a restart). + if external or mixed or gecis_context.foreign(req): try: normalized = gecis_context.normalize(req) except gecis_context.TransferError as e: self._transfer_begin() return self._transfer_fail(str(e)) # Small returning requests need no summarizer or local worker. - if external or gecis_context.tokens(normalized) > int(32768 * 0.70): + # Native opaque state must remain on its owner route. Its byte + # size is not a semantic token budget and cannot trigger a + # foreign summarizer, including when transfer state is cold. + if external or (mixed and not gecis_context.native_opaque(normalized) + and gecis_context.tokens(normalized) > int(32768 * 0.70)): prepared, started = self._prepare(normalized, source) if prepared is None: return diff --git a/apps/desktop/assets/codex-bridge/compression_engine.py b/apps/desktop/assets/codex-bridge/compression_engine.py index cb69cf6..0855adb 100644 --- a/apps/desktop/assets/codex-bridge/compression_engine.py +++ b/apps/desktop/assets/codex-bridge/compression_engine.py @@ -180,9 +180,13 @@ def prepare(handler, module, payload, checkpoints): slug = original.get('model', '') external = module.ek_model_mu(slug) source_model = payload.get('source_model') or '' + if not external and gc.native_opaque(original): + # Defense in depth for direct/older splitter callers. Native encrypted + # state never enters owner decoding or cross-provider summarization. + return {'request': gc.normalize(original)} # Defense in depth: an accidental worker call cannot alter native-only history. - if not external and not module.ek_model_mu(source_model) and not gc.foreign(original): - return {'request': copy.deepcopy(original), 'window': handler._transfer_window(slug)} + if not external and not module.ek_model_mu(source_model) and not gc.foreign_history(original): + return {'request': gc.normalize(original)} settings = settings_store.read(module.HOME) actual = (module.yonlendirilen(str(original.get('prompt_cache_key') or ''), slug) or slug) if external else slug window = handler._transfer_window(actual) diff --git a/apps/desktop/assets/codex-bridge/gecis_context.py b/apps/desktop/assets/codex-bridge/gecis_context.py index 7c8e3f8..1543409 100644 --- a/apps/desktop/assets/codex-bridge/gecis_context.py +++ b/apps/desktop/assets/codex-bridge/gecis_context.py @@ -54,6 +54,22 @@ def foreign(req): return False +def native_opaque(req): + """Native encrypted state belongs to the subscription, even in mixed history.""" + return any(isinstance(item, dict) and item.get("type") == "compaction" and item.get("encrypted_content") + and not str(item["encrypted_content"]).startswith(ozet_kopru.ISARET) + for item in req.get("input") or []) + + +def foreign_history(req): + """Semantic migration evidence, excluding tool ID format mismatches.""" + return any(isinstance(item, dict) and ( + "chatcmpl-" in str(item.get("id") or "") + or "astrabridge" in str(item.get("id") or "") + or str(item.get("encrypted_content") or "").startswith(ozet_kopru.ISARET)) + for item in req.get("input") or []) + + def normalize(req): """Repair local wrappers/IDs, retaining real OpenAI encrypted compaction. @@ -82,6 +98,10 @@ def normalize(req): if any(visible): result.append(ozet_kopru._ozet_mesaji("\n".join(visible))) continue + if ("chatcmpl-" in str(identity) or "astrabridge" in str(identity)) and ( + item.get("type") == "message" or item.get("role")): + # Inline message content is portable; its foreign server ID is not. + item.pop("id", None) prefix = PREFIXES.get(item.get("type")) if prefix and isinstance(identity, str) and identity and not identity.startswith(prefix + "_"): item["id"] = prefix + "_" + identity.split("_", 1)[-1] diff --git a/apps/desktop/assets/codex-bridge/kopru.py b/apps/desktop/assets/codex-bridge/kopru.py index dce84a2..49c29f0 100644 --- a/apps/desktop/assets/codex-bridge/kopru.py +++ b/apps/desktop/assets/codex-bridge/kopru.py @@ -40,7 +40,7 @@ from bekci_watch import MetadataFilter # Installed only by owned native launcher, never an HTTP control endpoint. WATCH_OWNER = None -SURUM = "16" +SURUM = "17" HERE = os.path.dirname(os.path.abspath(__file__)) try: with open(os.path.abspath(__file__), "rb") as _f: @@ -1287,7 +1287,7 @@ class H(http.server.BaseHTTPRequestHandler): for model in json.loads(cached["body"]).get("models", []): if model.get("slug") == slug and model.get("context_window"): return int(model["context_window"]) - return 32768 + raise gecis_context.TransferError("Abonelik modelinin bağlam kapasitesi katalogda bulunamadı; geçmiş korundu.") def _prepare_transfer(self, payload): diff --git a/apps/desktop/assets/codex-bridge/test_native_switch_regression.py b/apps/desktop/assets/codex-bridge/test_native_switch_regression.py new file mode 100644 index 0000000..db7f85a --- /dev/null +++ b/apps/desktop/assets/codex-bridge/test_native_switch_regression.py @@ -0,0 +1,83 @@ +"""Native switch regressions using the real HTTP splitter and offline peers.""" +import copy +import gzip +import json +import unittest +import ast +from pathlib import Path + +import bridge +import gecis_context as gc +import ozet_kopru +import test_ayirici as fixtures + + +class NativeSwitch(unittest.TestCase): + setUp = fixtures.Ayirici.setUp + tearDown = fixtures.Ayirici.tearDown + iste = fixtures.Ayirici.iste + + def test_unknown_native_capacity_is_not_invented(self): + tree = ast.parse(Path(__file__).with_name('kopru.py').read_text(encoding='utf-8')) + method = next(node for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == '_transfer_window') + ns = {'ek_model_mu': lambda _: False, 'LAST_MODELS': {}, 'gecis_context': gc, 'json': json} + exec(compile(ast.Module(body=[method], type_ignores=[]), 'window', 'exec'), ns) + from types import SimpleNamespace + with self.assertRaises(gc.TransferError): + ns['_transfer_window'](SimpleNamespace(headers={}), 'gpt-6-astra') + + def test_opaque_native_switch_with_persisted_foreign_residue(self): + self.ozgun.cevap = fixtures.duz(200, b'data: {"type":"response.completed"}\n\n', 'text/event-stream') + compact = {'type': 'compaction', 'id': 'cmp_native', + 'encrypted_content': 'gAAAAABp' + 'AqZ09_-jX' * 10000} + req = {'model': 'gpt-6.1-sol', 'prompt_cache_key': 'restart', 'input': [ + ozet_kopru.ozet_ogesi('DECISION-4827 must survive'), compact, + {'type': 'custom_tool_call', 'id': 'fc_123', 'call_id': 'call_123', + 'name': 'apply_patch', 'input': 'patch text'}, + {'type': 'custom_tool_call_output', 'id': 'fco_123', + 'call_id': 'call_123', 'output': 'complete'}, + {'role': 'user', 'content': 'continue'}]} + before = copy.deepcopy(req) + for cold, model in ((True, 'gpt-6.1-sol'), (False, 'gpt-6-astra'), (True, 'gpt-6-astra')): + if cold: + bridge.TRANSFERS.clear() + req['model'] = model + body = gzip.compress(json.dumps(req).encode()) + self.assertEqual(self.iste('POST', '/backend-api/codex/responses', body, + {'Content-Encoding': 'gzip'})[0], 200) + sent = json.loads(self.ozgun.gelen[-1]['govde']) + self.assertEqual(sent['input'][1], compact) + self.assertEqual(sent['input'][2]['id'], 'ctc_123') + self.assertEqual(sent['input'][3]['id'], 'ctco_123') + self.assertEqual(sent['input'][3]['call_id'], 'call_123') + self.assertIn('DECISION-4827', json.dumps(sent)) + self.assertEqual(self.kopru.gelen, []) + self.assertEqual(req['input'], before['input']) + + def test_clean_native_body_is_byte_identical_even_after_external_source(self): + req = {'model': 'ide/test', 'prompt_cache_key': 'return', 'input': []} + self.iste('POST', '/backend-api/codex/responses', json.dumps(req).encode()) + self.kopru.gelen.clear() + req.update(model='gpt-6-astra', input=[{'type': 'compaction', + 'encrypted_content': 'gAAAAABp' + 'X' * 100000}]) + raw = gzip.compress(json.dumps(req, indent=3).encode()) + self.iste('POST', '/backend-api/codex/responses', raw, {'Content-Encoding': 'gzip'}) + self.assertEqual(self.ozgun.gelen[-1]['govde'], raw) + self.assertEqual(self.kopru.gelen, []) + + def test_cold_native_tool_id_repair_does_not_require_semantic_budget(self): + req = {'model': 'gpt-6-astra', 'input': [ + {'type': 'custom_tool_call', 'id': 'fc_old', 'call_id': 'call_old', + 'name': 'apply_patch', 'input': 'x' * 100000}, + {'type': 'custom_tool_call_output', 'id': 'fco_old', + 'call_id': 'call_old', 'output': 'complete'}]} + self.iste('POST', '/backend-api/codex/responses', json.dumps(req).encode()) + self.assertEqual(len(self.kopru.gelen), 0) + sent = json.loads(self.ozgun.gelen[-1]['govde']) + self.assertEqual(sent, gc.normalize(req)) + self.assertEqual(req['input'][0]['id'], 'fc_old') + + +if __name__ == '__main__': + unittest.main() diff --git a/apps/desktop/electron/codex-link-config.ts b/apps/desktop/electron/codex-link-config.ts index 7097361..f6a13ba 100644 --- a/apps/desktop/electron/codex-link-config.ts +++ b/apps/desktop/electron/codex-link-config.ts @@ -27,7 +27,7 @@ export const CODEX_LINK_REFRESH_URL = `${CODEX_LINK_ORIGIN}/aiturk/codex-baglama export const CODEX_LINK_MARKER = '# AITURK IDE Codex Baglama' // Must equal SURUM in assets/codex-bridge/bridge.py; a running bridge that reports // another version is restarted so an updated IDE never keeps an old bridge alive. -export const CODEX_BRIDGE_VERSION = '16' +export const CODEX_BRIDGE_VERSION = '17' export const CODEX_LINK_LINE = `openai_base_url = "${CODEX_LINK_ORIGIN}/backend-api/codex" ${CODEX_LINK_MARKER}` export const CODEX_LINK_RUN_KEY = 'HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Run' diff --git a/apps/desktop/package.json b/apps/desktop/package.json index d9b7737..08b89fd 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -2,7 +2,7 @@ "name": "aiturk-ide", "productName": "AITURK IDE", "private": true, - "version": "1.0.0-beta.35", + "version": "1.0.0-beta.36", "description": "TurkServis resmi yapay zekâ geliştirme ortamı. Hermes Agent tabanlıdır.", "author": "AITURK / TurkServis", "repository": { diff --git a/package-lock.json b/package-lock.json index 5573052..a14ba37 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,7 +65,7 @@ }, "apps/desktop": { "name": "aiturk-ide", - "version": "1.0.0-beta.35", + "version": "1.0.0-beta.36", "license": "MIT", "dependencies": { "@assistant-ui/core": "0.2.23", diff --git a/tests/tools/test_search_giant_line_containment.py b/tests/tools/test_search_giant_line_containment.py index a31c650..658fc5d 100644 --- a/tests/tools/test_search_giant_line_containment.py +++ b/tests/tools/test_search_giant_line_containment.py @@ -20,6 +20,7 @@ import subprocess import pytest from tools.file_operations import ShellFileOperations +from tools.environments.local import LocalEnvironment # Big enough to prove containment, small enough to keep the test fast. GIANT = 5 * 1024 * 1024 # 5MB single line @@ -27,22 +28,20 @@ GIANT = 5 * 1024 * 1024 # 5MB single line STDOUT_CEILING = 1 * 1024 * 1024 -class RecordingEnv: +class RecordingEnv(LocalEnvironment): """Local bash executor that records the largest stdout it returned.""" def __init__(self, cwd): - self.cwd = cwd + super().__init__(cwd=cwd) self.max_stdout = 0 def execute(self, command, timeout=60, **kwargs): - proc = subprocess.run( - ["bash", "-c", command], - capture_output=True, text=True, errors="replace", - timeout=timeout + 30, - ) - out = proc.stdout + (proc.stderr or "") + # Use the application's shell selection/path handling. On Windows a + # bare 'bash' may select WSL, whose filesystem is not the native one. + result = super().execute(command, timeout=timeout + 30, **kwargs) + out = result.get("output", "") self.max_stdout = max(self.max_stdout, len(out)) - return {"output": out, "returncode": proc.returncode} + return result @pytest.fixture() @@ -58,6 +57,8 @@ def _ops(giant_dir, engine): env = RecordingEnv(str(giant_dir)) ops = ShellFileOperations(env) ops._has_command = lambda cmd: cmd == engine + resolve = ops._resolve_command + ops._resolve_command = lambda cmd: None if cmd == "rg" and engine != "rg" else resolve(cmd) return ops, env @@ -96,5 +97,5 @@ def test_line_cap_skipped_for_path_and_count_modes(giant_dir, engine, output_mod if output_mode == "files_only": assert {os.path.basename(f) for f in result.files} == {"trace.json", "small.py"} else: - assert {os.path.basename(k) for k in result.counts} == {"trace.json", "small.py"} + assert {os.path.basename(k) for k, count in result.counts.items() if count} == {"trace.json", "small.py"} assert env.max_stdout < STDOUT_CEILING diff --git a/tests/tools/test_search_project_scope.py b/tests/tools/test_search_project_scope.py new file mode 100644 index 0000000..463f86f --- /dev/null +++ b/tests/tools/test_search_project_scope.py @@ -0,0 +1,112 @@ +"""Project scope and resource limits: real local paths, no broad subprocesses.""" +from unittest.mock import Mock + +from tools.file_operations import ShellFileOperations, SearchResult, ExecuteResult +from tools.environments.local import LocalEnvironment + + +def projects(tmp_path): + for name in ('one', 'two'): + project = tmp_path / name + project.mkdir() + (project / '.git').mkdir() + (project / 'source.py').write_text('needle\n') + return tmp_path / 'one' + + +def test_ambiguous_scope_starts_no_process(tmp_path): + projects(tmp_path) + ops = ShellFileOperations(LocalEnvironment(str(tmp_path))) + ops._exec = Mock(side_effect=AssertionError('must not start process')) + result = ops.search('needle', str(tmp_path)) + assert 'scope' in result.error.lower() + assert 'one' in result.error and 'two' in result.error + ops._exec.assert_not_called() + + +def test_umbrella_uses_cwd_project_and_reports_it(tmp_path): + project = projects(tmp_path) + ops = ShellFileOperations(LocalEnvironment(str(project))) + ops._exec = Mock(return_value=ExecuteResult(stdout='exists', exit_code=0)) + ops._search_content = Mock(return_value=SearchResult()) + result = ops.search('needle', str(tmp_path)) + assert ops._search_content.call_args.args[1] == str(project) + assert str(project) in result.warning + + +def test_broad_opt_in_and_explicit_file(tmp_path): + project = projects(tmp_path) + ops = ShellFileOperations(LocalEnvironment(str(project))) + ops._exec = Mock(return_value=ExecuteResult(stdout='exists', exit_code=0)) + ops._search_content = Mock(return_value=SearchResult()) + for root, broad in ((tmp_path, True), (project / 'source.py', False)): + assert not ops.search('needle', str(root), allow_broad=broad).error + assert ops._search_content.call_args.args[1] == str(root) + + +def test_rg_commands_bound_threads_and_prune(tmp_path): + project = projects(tmp_path) + ops = ShellFileOperations(LocalEnvironment(str(project))) + ops._exec = Mock(return_value=ExecuteResult(stdout='', exit_code=1)) + ops._search_with_rg('needle', str(project), '*.py', 50, 0, 'content', 0, rg_executable='rg') + command = ops._exec.call_args.args[0] + assert '--threads 2' in command + assert 'node_modules' in command and '.git' in command + assert '*.py' in command + ops._search_files_rg('*.py', str(project), 50, 0, rg_executable='rg') + command = ops._exec.call_args.args[0] + assert '--threads 2' in command and 'node_modules' in command + + +def test_concurrent_searches_have_two_slots_and_release_after_failure(tmp_path): + import threading + from concurrent.futures import ThreadPoolExecutor + project = projects(tmp_path) + lock = threading.Lock() + release = threading.Event() + entered = threading.Event() + active = peak = 0 + + def run_search(*args): + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + if active == 2: + entered.set() + try: + assert release.wait(5) + return SearchResult() + finally: + with lock: + active -= 1 + + with ThreadPoolExecutor(max_workers=6) as pool: + pending = [] + for _ in range(6): + ops = ShellFileOperations(LocalEnvironment(str(project))) + ops._search_scoped = run_search + pending.append(pool.submit(ops.search, 'needle')) + try: + assert entered.wait(3) + assert peak == 2 + finally: + release.set() + assert all(not task.result().error for task in pending) + assert peak == 2 + + +def test_real_rg_project_content_and_nested_glob(tmp_path): + project = projects(tmp_path) + (project / 'src').mkdir() + (project / 'src' / 'nested.py').write_text('needle\n') + (project / 'node_modules').mkdir() + (project / 'node_modules' / 'noise.py').write_text('needle\n') + ops = ShellFileOperations(LocalEnvironment(str(project))) + result = ops.search('needle', str(tmp_path), file_glob='*.py') + assert not result.error + assert len(result.matches) == 2 + assert all('node_modules' not in item.path for item in result.matches) + result = ops.search('src/*.py', str(project), target='files') + assert not result.error + assert len(result.files) == 1 and 'nested.py' in result.files[0] diff --git a/tools/file_operations.py b/tools/file_operations.py index a349e9a..9738904 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -50,7 +50,8 @@ from agent.file_safety import ( get_write_denied_error, is_write_denied as _shared_is_write_denied, ) -from agent.search_policy import SEARCH_PRUNE_DIR_NAMES +from agent.search_policy import (SEARCH_PRUNE_DIR_NAMES, SEARCH_RG_THREADS, + SEARCH_MAX_CONCURRENT, local_search_scope) from tools import interrupt as tool_interrupt logger = logging.getLogger(__name__) @@ -73,6 +74,8 @@ _MACOS_TCC_PROTECTED_HOME_DIRS = ( ) +_SEARCH_SLOTS = threading.BoundedSemaphore(SEARCH_MAX_CONCURRENT) + _FILENAME_SEARCH_ADMISSION = threading.Condition() _ACTIVE_FILENAME_SEARCH_ROOTS: set[tuple[str, str, str]] = set() _FILENAME_SEARCH_WAIT_SECONDS = 0.05 @@ -3432,7 +3435,31 @@ class ShellFileOperations(FileOperations): def search(self, pattern: str, path: str = ".", target: str = "content", file_glob: Optional[str] = None, limit: int = 50, offset: int = 0, output_mode: str = "content", context: int = 0, - order: str = "discovery") -> SearchResult: + order: str = "discovery", allow_broad: bool = False) -> SearchResult: + # One admission slot covers discovery, traversal and zero-match probes. + # It is process-wide across file-ops instances, and interruptible. + while not _SEARCH_SLOTS.acquire(timeout=0.05): + if tool_interrupt.is_interrupted(): + return SearchResult(error="Search interrupted while waiting for a search slot.") + try: + if tool_interrupt.is_interrupted(): + return SearchResult(error="Search interrupted.") + return self._search_scoped(pattern, path, target, file_glob, limit, + offset, output_mode, context, order, allow_broad) + finally: + _SEARCH_SLOTS.release() + + def _project_search_scope(self, path, allow_broad=False): + from tools.environments.local import LocalEnvironment, _msys_to_windows_path + if allow_broad or not isinstance(self.env, LocalEnvironment): + return path, None, None + cwd = getattr(self.env, "cwd", None) or self.cwd + return local_search_scope(_msys_to_windows_path(path), cwd, _HOME) + + def _search_scoped(self, pattern: str, path: str = ".", target: str = "content", + file_glob: Optional[str] = None, limit: int = 50, offset: int = 0, + output_mode: str = "content", context: int = 0, + order: str = "discovery", allow_broad: bool = False) -> SearchResult: """ Search for content or files. @@ -3460,6 +3487,9 @@ class ShellFileOperations(FileOperations): # Expand ~ and other shell paths path = self._expand_path(path) + path, scope_notice, scope_error = self._project_search_scope(path, allow_broad) + if scope_error: + return SearchResult(error=scope_error) # Validate that the path exists before searching check = self._exec(f"test -e {self._escape_shell_arg(path)} && echo exists || echo not_found") @@ -3470,7 +3500,7 @@ class ShellFileOperations(FileOperations): # merge the results, and report the skipped parts. multi = self._try_multi_path_search( pattern, path, target, file_glob, limit, offset, output_mode, context, - order, + order, allow_broad, ) if multi is not None: return multi @@ -3518,6 +3548,8 @@ class ShellFileOperations(FileOperations): f"an unattended privacy prompt: {skipped}. Search a protected " "folder directly when access is intentional." ) + if scope_notice: + result.warning = " ".join(filter(None, [scope_notice, result.warning])) return result def _macos_search_exclusions(self, path: str) -> List[str]: @@ -3587,7 +3619,7 @@ class ShellFileOperations(FileOperations): def _try_multi_path_search(self, pattern: str, path: str, target: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int, - order: str = "discovery") -> Optional[SearchResult]: + order: str = "discovery", allow_broad: bool = False) -> Optional[SearchResult]: """Recover a not-found ``path`` that is really several paths in one string. Production trajectories show models passing "dir1 dir2 dir3" (or @@ -3604,9 +3636,14 @@ class ShellFileOperations(FileOperations): parts = path.split() if len(parts) < 2: return None - existing, missing = [], [] + existing, missing, scope_notices = [], [], [] for p in parts: expanded = self._expand_path(p) + expanded, notice, error = self._project_search_scope(expanded, allow_broad) + if error: + return SearchResult(error=error) + if notice: + scope_notices.append(notice) chk = self._exec( f"test -e {self._escape_shell_arg(expanded)} && echo exists || echo not_found" ) @@ -3641,7 +3678,7 @@ class ShellFileOperations(FileOperations): note += "; skipped missing: " + ", ".join(missing[:3]) if len(missing) > 3: note += f" (+{len(missing) - 3} more)" - warning_parts = [note] + warning_parts = [note, *scope_notices] if not merged.error: protected_paths = [ absolute @@ -3704,7 +3741,7 @@ class ShellFileOperations(FileOperations): glob_expr = f" --glob {self._escape_shell_arg(file_glob)}" if file_glob else "" probe = self._exec( - f"{rg} -i --count-matches{glob_expr} " + f"{rg} --threads {SEARCH_RG_THREADS} -i --count-matches{glob_expr} {self._search_prune_glob_args()} " f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " f"2>/dev/null | head -50", timeout=30, @@ -3723,7 +3760,7 @@ class ShellFileOperations(FileOperations): # files remain diagnosable, but prune heavyweight trees before rg can # recurse into them. hidden = self._exec( - f"{rg} --hidden --no-ignore --count-matches{glob_expr}" + f"{rg} --threads {SEARCH_RG_THREADS} --hidden --no-ignore --count-matches{glob_expr}" f" {self._search_prune_glob_args()} " f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " f"2>/dev/null | head -50", @@ -3738,7 +3775,7 @@ class ShellFileOperations(FileOperations): ) if re.search(r"[.\[\](){}?*+^$\\|]", pattern): fixed = self._exec( - f"{rg} -F --count-matches{glob_expr} " + f"{rg} --threads {SEARCH_RG_THREADS} -F --count-matches{glob_expr} {self._search_prune_glob_args()} " f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " f"2>/dev/null | head -50", timeout=30, @@ -3813,7 +3850,7 @@ class ShellFileOperations(FileOperations): )) try: return self._search_files_rg( - search_pattern, path, limit, offset, order, + pattern, path, limit, offset, order, rg_executable=rg_executable, ) finally: @@ -3848,6 +3885,9 @@ class ShellFileOperations(FileOperations): hidden_prune = ( f" \\( -type d -name '.*'{root_exemptions} \\) -prune -o" ) + names = " -o ".join(f"-name {self._escape_shell_arg(name)}" + for name in sorted(SEARCH_PRUNE_DIR_NAMES)) + hidden_prune += f" \\( -type d \\( {names} \\){root_exemptions} \\) -prune -o" protected_paths = [ absolute for _root, _relative, absolute @@ -3986,7 +4026,7 @@ class ShellFileOperations(FileOperations): f"cd {self._escape_shell_arg(scoped_common)} && " if scoped_common else "" ) cmd = ( - f"set -o pipefail; {cd_prefix}{rg} --files{sort_arg} -g {self._escape_shell_arg(glob_pattern)}" + f"set -o pipefail; {cd_prefix}{rg} --files{sort_arg} --threads {SEARCH_RG_THREADS} -g {self._escape_shell_arg(glob_pattern)} {self._search_prune_glob_args()}" f"{exclusion_args} -- {root_args} 2>/dev/null | head -n {fetch_limit}" ) result = self._exec(cmd, timeout=60) @@ -4062,7 +4102,7 @@ class ShellFileOperations(FileOperations): rg_executable = rg_executable or self._resolve_command("rg") if not rg_executable: return SearchResult(error="Content search requires ripgrep (rg).") - cmd_parts = [self._quote_executable(rg_executable), "--line-number", "--no-heading", "--with-filename"] + cmd_parts = [self._quote_executable(rg_executable), "--threads", str(SEARCH_RG_THREADS), "--line-number", "--no-heading", "--with-filename"] # Giant-single-line containment (ported from cline/cline#13525): a # match inside a serialized dump (multi-MB single-line JSON/minified @@ -4100,6 +4140,8 @@ class ShellFileOperations(FileOperations): if file_glob: cmd_parts.extend(["--glob", self._escape_shell_arg(file_glob)]) + cmd_parts.append(self._search_prune_glob_args()) + # Output mode handling if output_mode == "files_only": cmd_parts.append("-l") # Files only @@ -4230,6 +4272,8 @@ class ShellFileOperations(FileOperations): # Exclude hidden directories (matching ripgrep's default behavior). # This prevents searching inside .hub/index-cache/, .git/, etc. cmd_parts.append("--exclude-dir='.*'") + cmd_parts.extend(f"--exclude-dir={self._escape_shell_arg(name)}" + for name in sorted(SEARCH_PRUNE_DIR_NAMES)) # Protected-dir pruning CANNOT use --exclude-dir here: grep matches # exclude-dir globs against BASENAMES anywhere in the tree, so diff --git a/tools/file_tools.py b/tools/file_tools.py index 06fc26a..db465d6 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -2653,7 +2653,7 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", file_glob: str = None, limit: int = 50, offset: int = 0, output_mode: str = "content", context: int = 0, order: str = "discovery", - task_id: str = "default") -> str: + task_id: str = "default", allow_broad: bool = False) -> str: """Search for content or files.""" try: offset, limit = normalize_search_pagination(offset, limit) @@ -2670,6 +2670,7 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", limit, offset, order, + allow_broad, ) with _read_tracker_lock: task_data = _read_tracker.setdefault(task_id, { @@ -2717,6 +2718,7 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", pattern=pattern, path=path, target=target, file_glob=file_glob, limit=limit, offset=offset, output_mode=output_mode, context=context, order=order, + **({"allow_broad": True} if allow_broad else {}), ) omitted = _filter_read_blocked_search_results(result, task_id) if hasattr(result, 'matches'): @@ -2912,6 +2914,7 @@ SEARCH_FILES_SCHEMA = { "parameters": { "type": "object", "properties": { + "allow_broad": {"type": "boolean", "default": False, "description": "Deliberately search a home, drive or multi-project umbrella. Default scopes to the current project or asks for a scope. Does not remove pruning or CPU limits."}, "pattern": {"type": "string", "description": "Regex pattern for content search, or glob pattern (e.g., '*.py') for file search"}, "target": {"type": "string", "enum": ["content", "files"], "description": "'content' searches inside file contents, 'files' searches for files by name", "default": "content"}, "path": {"type": "string", "description": "Directory or file to search in (default: current working directory)", "default": "."}, @@ -2979,7 +2982,8 @@ def _handle_search_files(args, **kw): pattern=args.get("pattern", ""), target=target, path=args.get("path", "."), file_glob=args.get("file_glob"), limit=args.get("limit", 50), offset=args.get("offset", 0), output_mode=args.get("output_mode", "content"), context=args.get("context", 0), - order=args.get("order", "discovery"), task_id=tid) + order=args.get("order", "discovery"), task_id=tid, + allow_broad=args.get("allow_broad") is True) def _read_file_schema_overrides():