diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 8f021fe..600d153 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -17,6 +17,9 @@ asyncio_mode = "auto" [tool.ruff] line-length = 100 target-version = "py311" +# Generated by protoc; any fix here is overwritten on the next `protoc` run, so linting it +# would produce a permanently dirty tree. +extend-exclude = ["src/core/indexer/indexer_pb2.py", "src/core/indexer/indexer_pb2_grpc.py"] [tool.ruff.lint] select = ["E", "F", "I", "W"] diff --git a/apps/api/src/core/indexer/__init__.py b/apps/api/src/core/indexer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/src/core/indexer/client.py b/apps/api/src/core/indexer/client.py new file mode 100644 index 0000000..19ae6b1 --- /dev/null +++ b/apps/api/src/core/indexer/client.py @@ -0,0 +1,159 @@ +""" +Async client for the Go indexer service. + +Optional and off by default: the Python walk-and-parse path in IndexingService remains the +default, and this is used only when indexer_grpc_enabled is set. Both produce the same +chunk shape, so the caller does not branch beyond choosing a source. + +Uses grpc.aio, which grpc documents as stable, and grpcio is already an installed +dependency (transitively via chromadb) so enabling this adds no new runtime wheel. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import AsyncIterator, List, Optional + +logger = logging.getLogger(__name__) + + +@dataclass +class RemoteChunk: + """Mirrors the Chunk message. Deliberately not the ORM model: this is wire data.""" + file_path: str + language: str + chunk_type: str + name: str + content: str + start_line: int + end_line: int + had_parse_error: bool + + +@dataclass +class IndexProgress: + stage: str + current_path: str + files_processed: int + total_files: int + percent: float + + +@dataclass +class IndexSummary: + files_walked: int + files_parsed: int + files_skipped: int + chunks_emitted: int + files_with_errors: int + duration_ms: int + + +class IndexerUnavailable(RuntimeError): + """Raised when the service cannot be reached, so the caller can fall back to Python.""" + + +class IndexerClient: + def __init__(self, target: str, timeout_seconds: float = 900.0): + self._target = target + self._timeout = timeout_seconds + + async def health(self) -> Optional[dict]: + """Returns the service's linked grammars, or None when unreachable.""" + import grpc + + from src.core.indexer import indexer_pb2, indexer_pb2_grpc + + try: + async with grpc.aio.insecure_channel(self._target) as channel: + stub = indexer_pb2_grpc.IndexerStub(channel) + res = await stub.Health(indexer_pb2.HealthRequest(), timeout=10) + return { + "ok": res.ok, + "version": res.version, + # Reported rather than assumed: the Go build links a narrower grammar + # set than the Python parser, and pretending otherwise would silently + # drop languages. + "parsers": {p.language: list(p.extensions) for p in res.parsers}, + } + except Exception as exc: + logger.warning("Indexer service health check failed (%s): %s", self._target, exc) + return None + + async def index_repo( + self, + repo_id: str, + root_path: str, + max_files: int = 0, + max_file_size_kb: int = 0, + batch_size: int = 0, + ) -> AsyncIterator[object]: + """ + Stream IndexProgress / List[RemoteChunk] / IndexSummary as the server produces them. + + Yields heterogeneous types on purpose: the caller wants progress promptly and + chunks in batches, and forcing both into one shape would mean buffering the whole + repository before anything is usable. + """ + import grpc + + from src.core.indexer import indexer_pb2, indexer_pb2_grpc + + request = indexer_pb2.IndexRequest( + repo_id=repo_id, + root_path=root_path, + max_files=max_files, + max_file_size_kb=max_file_size_kb, + batch_size=batch_size, + ) + + try: + async with grpc.aio.insecure_channel(self._target) as channel: + stub = indexer_pb2_grpc.IndexerStub(channel) + async for event in stub.IndexRepo(request, timeout=self._timeout): + which = event.WhichOneof("event") + if which == "progress": + p = event.progress + yield IndexProgress( + stage=p.stage, current_path=p.current_path, + files_processed=p.files_processed, total_files=p.total_files, + percent=p.percent, + ) + elif which == "chunks": + yield [ + RemoteChunk( + file_path=c.file_path, language=c.language, + chunk_type=c.chunk_type, name=c.name, content=c.content, + start_line=c.start_line, end_line=c.end_line, + had_parse_error=c.had_parse_error, + ) + for c in event.chunks.chunks + ] + elif which == "completed": + c = event.completed + yield IndexSummary( + files_walked=c.files_walked, files_parsed=c.files_parsed, + files_skipped=c.files_skipped, chunks_emitted=c.chunks_emitted, + files_with_errors=c.files_with_errors, duration_ms=c.duration_ms, + ) + elif which == "failed": + # A server-side failure arrives as a stream event, not a status + # code, so it has to be re-raised here to stop the caller treating + # a truncated stream as a complete index. + raise IndexerUnavailable( + f"indexer failed: {event.failed.message} ({event.failed.path})" + ) + except IndexerUnavailable: + raise + except Exception as exc: + raise IndexerUnavailable(f"indexer stream failed: {exc}") from exc + + +async def collect_chunks(client: IndexerClient, repo_id: str, root_path: str) -> List[RemoteChunk]: + """Convenience for tests and one-shot use; drains the stream into a list.""" + out: List[RemoteChunk] = [] + async for item in client.index_repo(repo_id, root_path): + if isinstance(item, list): + out.extend(item) + return out diff --git a/apps/api/src/core/indexer/indexer_pb2.py b/apps/api/src/core/indexer/indexer_pb2.py new file mode 100644 index 0000000..d30dd3f --- /dev/null +++ b/apps/api/src/core/indexer/indexer_pb2.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: indexer.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 1, + '', + 'indexer.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rindexer.proto\x12\x15\x63odebaseqa.indexer.v1\"s\n\x0cIndexRequest\x12\x0f\n\x07repo_id\x18\x01 \x01(\t\x12\x11\n\troot_path\x18\x02 \x01(\t\x12\x11\n\tmax_files\x18\x03 \x01(\r\x12\x18\n\x10max_file_size_kb\x18\x04 \x01(\r\x12\x12\n\nbatch_size\x18\x05 \x01(\r\"\xe7\x01\n\nIndexEvent\x12\x33\n\x08progress\x18\x01 \x01(\x0b\x32\x1f.codebaseqa.indexer.v1.ProgressH\x00\x12\x33\n\x06\x63hunks\x18\x02 \x01(\x0b\x32!.codebaseqa.indexer.v1.ChunkBatchH\x00\x12\x35\n\tcompleted\x18\x03 \x01(\x0b\x32 .codebaseqa.indexer.v1.CompletedH\x00\x12/\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32\x1d.codebaseqa.indexer.v1.FailedH\x00\x42\x07\n\x05\x65vent\"n\n\x08Progress\x12\r\n\x05stage\x18\x01 \x01(\t\x12\x14\n\x0c\x63urrent_path\x18\x02 \x01(\t\x12\x17\n\x0f\x66iles_processed\x18\x03 \x01(\r\x12\x13\n\x0btotal_files\x18\x04 \x01(\r\x12\x0f\n\x07percent\x18\x05 \x01(\x01\":\n\nChunkBatch\x12,\n\x06\x63hunks\x18\x01 \x03(\x0b\x32\x1c.codebaseqa.indexer.v1.Chunk\"\x9e\x01\n\x05\x43hunk\x12\x11\n\tfile_path\x18\x01 \x01(\t\x12\x10\n\x08language\x18\x02 \x01(\t\x12\x12\n\nchunk_type\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\t\x12\x12\n\nstart_line\x18\x06 \x01(\r\x12\x10\n\x08\x65nd_line\x18\x07 \x01(\r\x12\x17\n\x0fhad_parse_error\x18\x08 \x01(\x08\"\x96\x01\n\tCompleted\x12\x14\n\x0c\x66iles_walked\x18\x01 \x01(\r\x12\x14\n\x0c\x66iles_parsed\x18\x02 \x01(\r\x12\x15\n\rfiles_skipped\x18\x03 \x01(\r\x12\x16\n\x0e\x63hunks_emitted\x18\x04 \x01(\r\x12\x19\n\x11\x66iles_with_errors\x18\x05 \x01(\r\x12\x13\n\x0b\x64uration_ms\x18\x06 \x01(\x03\"\'\n\x06\x46\x61iled\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"\x0f\n\rHealthRequest\"a\n\x0eHealthResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x32\n\x07parsers\x18\x03 \x03(\x0b\x32!.codebaseqa.indexer.v1.ParserInfo\"2\n\nParserInfo\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nextensions\x18\x02 \x03(\t2\xb7\x01\n\x07Indexer\x12U\n\tIndexRepo\x12#.codebaseqa.indexer.v1.IndexRequest\x1a!.codebaseqa.indexer.v1.IndexEvent0\x01\x12U\n\x06Health\x12$.codebaseqa.indexer.v1.HealthRequest\x1a%.codebaseqa.indexer.v1.HealthResponseB={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class IndexerStub: + """Repository indexing: file walk plus tree-sitter parsing, extracted from the Python API. + + WHY A SEPARATE SERVICE, STATED HONESTLY + Not for parse throughput. Measured on this repository, ThreadPoolExecutor over the + Python parser gives 1.02x (py-tree-sitter never releases the GIL) and + ProcessPoolExecutor gives 2.84x, while only ~41% of the parse phase is actually C -- so + a perfect native rewrite ceilings around 2.4x, below what a half-day change to the + existing Python already achieves. + + The real reason is a process and failure boundary. repos.py runs indexing by spinning a + new event loop inside an anyio threadpool thread and blocking it for minutes, in the + same process that serves chat. This moves that CPU-bound, GIL-bound, minutes-long stage + behind an explicit contract. + + Server streaming is the point of using gRPC here rather than one big response: progress + becomes part of the contract instead of shared mutable state, which is the same bug + class that made the SSE progress bar show only 0% or 100%. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.IndexRepo = channel.unary_stream( + '/codebaseqa.indexer.v1.Indexer/IndexRepo', + request_serializer=indexer__pb2.IndexRequest.SerializeToString, + response_deserializer=indexer__pb2.IndexEvent.FromString, + _registered_method=True) + self.Health = channel.unary_unary( + '/codebaseqa.indexer.v1.Indexer/Health', + request_serializer=indexer__pb2.HealthRequest.SerializeToString, + response_deserializer=indexer__pb2.HealthResponse.FromString, + _registered_method=True) + + +class IndexerServicer: + """Repository indexing: file walk plus tree-sitter parsing, extracted from the Python API. + + WHY A SEPARATE SERVICE, STATED HONESTLY + Not for parse throughput. Measured on this repository, ThreadPoolExecutor over the + Python parser gives 1.02x (py-tree-sitter never releases the GIL) and + ProcessPoolExecutor gives 2.84x, while only ~41% of the parse phase is actually C -- so + a perfect native rewrite ceilings around 2.4x, below what a half-day change to the + existing Python already achieves. + + The real reason is a process and failure boundary. repos.py runs indexing by spinning a + new event loop inside an anyio threadpool thread and blocking it for minutes, in the + same process that serves chat. This moves that CPU-bound, GIL-bound, minutes-long stage + behind an explicit contract. + + Server streaming is the point of using gRPC here rather than one big response: progress + becomes part of the contract instead of shared mutable state, which is the same bug + class that made the SSE progress bar show only 0% or 100%. + """ + + def IndexRepo(self, request, context): + """Walk and parse a checkout, streaming progress and chunk batches as they are produced. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Health(self, request, context): + """Cheap liveness probe. Returns the grammars this build actually links, which is not + guaranteed to match the Python side -- see ParserInfo. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_IndexerServicer_to_server(servicer, server): + rpc_method_handlers = { + 'IndexRepo': grpc.unary_stream_rpc_method_handler( + servicer.IndexRepo, + request_deserializer=indexer__pb2.IndexRequest.FromString, + response_serializer=indexer__pb2.IndexEvent.SerializeToString, + ), + 'Health': grpc.unary_unary_rpc_method_handler( + servicer.Health, + request_deserializer=indexer__pb2.HealthRequest.FromString, + response_serializer=indexer__pb2.HealthResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'codebaseqa.indexer.v1.Indexer', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('codebaseqa.indexer.v1.Indexer', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class Indexer: + """Repository indexing: file walk plus tree-sitter parsing, extracted from the Python API. + + WHY A SEPARATE SERVICE, STATED HONESTLY + Not for parse throughput. Measured on this repository, ThreadPoolExecutor over the + Python parser gives 1.02x (py-tree-sitter never releases the GIL) and + ProcessPoolExecutor gives 2.84x, while only ~41% of the parse phase is actually C -- so + a perfect native rewrite ceilings around 2.4x, below what a half-day change to the + existing Python already achieves. + + The real reason is a process and failure boundary. repos.py runs indexing by spinning a + new event loop inside an anyio threadpool thread and blocking it for minutes, in the + same process that serves chat. This moves that CPU-bound, GIL-bound, minutes-long stage + behind an explicit contract. + + Server streaming is the point of using gRPC here rather than one big response: progress + becomes part of the contract instead of shared mutable state, which is the same bug + class that made the SSE progress bar show only 0% or 100%. + """ + + @staticmethod + def IndexRepo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/codebaseqa.indexer.v1.Indexer/IndexRepo', + indexer__pb2.IndexRequest.SerializeToString, + indexer__pb2.IndexEvent.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def Health(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/codebaseqa.indexer.v1.Indexer/Health', + indexer__pb2.HealthRequest.SerializeToString, + indexer__pb2.HealthResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/services/indexer/.gitignore b/services/indexer/.gitignore new file mode 100644 index 0000000..f041915 --- /dev/null +++ b/services/indexer/.gitignore @@ -0,0 +1,2 @@ +# Built binary; rebuild with `CGO_ENABLED=1 go build -o bin/indexer .` +bin/ diff --git a/services/indexer/README.md b/services/indexer/README.md new file mode 100644 index 0000000..7b21a25 --- /dev/null +++ b/services/indexer/README.md @@ -0,0 +1,113 @@ +# Indexer service (Go + gRPC) + +Walk-and-parse stage extracted from the Python API, streaming results over gRPC. + +## Why this exists — and why it is *not* about speed + +The obvious claim ("rewrote the parser in Go for performance") is false, and it was +measured on this repository before any Go was written: + +``` +files parsed : 900 +sequential : 1115.3 ms (1.00x) +ThreadPool(4) : 1092.5 ms (1.02x) <- threads give nothing +ProcessPool(4) : 393.3 ms (2.84x) +``` + +`py-tree-sitter` never releases the GIL, so threads are ~1.0x — which also means the +obvious `anyio.to_thread` fix would not have worked. Only ~41% of the parse phase is +actually C, so a perfect native rewrite ceilings around **2.4x**, *below* the 2.84x that +`ProcessPoolExecutor` on the existing Python already delivers for about half a day's work. + +The real reason is a **process and failure boundary**. `repos.py` runs indexing by spinning +a new event loop inside an anyio threadpool thread and blocking it for minutes, in the same +process that serves chat. This moves that CPU-bound, GIL-bound, minutes-long stage behind +an explicit typed contract. + +Server streaming is why gRPC rather than one large response: progress becomes part of the +contract instead of shared mutable state — the same bug class that made the SSE progress +bar show only 0% or 100%. + +## Binding choice + +Uses the **official** `tree-sitter/go-tree-sitter`. Not `smacker/go-tree-sitter`, which is +the trap: it has roughly twice the stars (562 vs 288) and outranks the official binding in +search results, but was last pushed **2024-08-27** with 42 open issues and no deprecation +notice. + +## Two bugs not repeated here + +Both were found and fixed in the Python parser; this implementation gets them right from +the start, and there are tests for each. + +- **`.tsx` uses `LanguageTSX()`**, not `LanguageTypescript()`. The plain TypeScript grammar + cannot parse JSX and returns a tree full of ERROR nodes for every `.tsx` file. +- **Chunk text comes from `node.Utf8Text(source)`** over the source *bytes*. tree-sitter + offsets are byte offsets; slicing a decoded string with them corrupts every chunk after + the first multi-byte character. + +`had_parse_error` is on the wire because tree-sitter does not fail on a syntax error — it +returns a partial tree. Without that flag the Python caller cannot decide to fall back to +raw indexing. + +## Build and run + +```bash +cd services/indexer +CGO_ENABLED=1 go build -o bin/indexer . +./bin/indexer -addr :50051 +``` + +`CGO_ENABLED=1` is mandatory: every grammar is cgo. That means a C toolchain in any builder +image, no `scratch` base, and a slower uncached build than the all-wheels Python image. + +## Grammar scope, stated rather than implied + +This build links **5** grammars (python, javascript, typescript, tsx, go) against the +Python side's 9. Each grammar is a separate generated `parser.c`, so each one costs build +time and binary size. `Health()` reports exactly what is linked so the caller never has to +assume parity: + +``` +linked grammars: {'go': ['.go'], 'javascript': ['.js', '.jsx'], + 'python': ['.py'], 'tsx': ['.tsx'], 'typescript': ['.ts']} +``` + +## One fixed bug in the port + +The Python `_find_files` max-files cap did not work: its `break` left only the inner +filename loop, so `os.walk` continued into the next directory and kept appending. The Go +walker uses `fs.SkipAll`, which actually stops, and reports `Truncated` so a partial index +is not mistaken for full coverage. + +## Verified end to end + +Go server running, Python client streaming, against this repository's `apps/api/src`: + +``` +health: True v0.1.0 +progress events : 5 +chunk batches : 3 +chunks received : 509 +summary : walked=59 parsed=58 chunks=509 errors=0 99ms +chunk kinds : {'module': 17, 'class': 121, 'function': 109, 'method': 262} +example : class 'Level' in api/graphql/schema.py L47-53 +``` + +Correctness cases: + +``` +tsx function 'Widget' had_parse_error=False +function 'after_unicode' L2-3 content starts: 'def after_unicode(x):' +class 'Café' L4-6 content starts: 'class Café:' +method 'método' L5-6 content starts: 'def método(self):' +``` + +## Not done + +The service is **not wired into the indexing pipeline** — `IndexingService` still uses the +Python parser, and `IndexerClient` is opt-in. Doing that swap needs a decision this PR does +not make: the service must be co-located with the API on the shared volume, because clones +live under `./data/repos//` and a Fly/EBS-style volume attaches to exactly one +machine. There is also no CI job for Go yet (`ci.yml` has Python 3.11 and Node 20 only, no +cgo toolchain, no grammar-compile caching, no cross-language contract test). diff --git a/services/indexer/gen/indexer.pb.go b/services/indexer/gen/indexer.pb.go new file mode 100644 index 0000000..f78fc6e --- /dev/null +++ b/services/indexer/gen/indexer.pb.go @@ -0,0 +1,867 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc v5.29.3 +// source: indexer.proto + +package gen + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type IndexRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RepoId string `protobuf:"bytes,1,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` + // Absolute path to an existing checkout. The service does NOT clone: the API already + // owns credentials and the clone lives on a volume attached to one machine, so cloning + // here would duplicate auth handling and force the two to share a filesystem for a + // reason other than reading it. + RootPath string `protobuf:"bytes,2,opt,name=root_path,json=rootPath,proto3" json:"root_path,omitempty"` + // 0 means "use the server default" so the API can defer rather than restate policy. + MaxFiles uint32 `protobuf:"varint,3,opt,name=max_files,json=maxFiles,proto3" json:"max_files,omitempty"` + MaxFileSizeKb uint32 `protobuf:"varint,4,opt,name=max_file_size_kb,json=maxFileSizeKb,proto3" json:"max_file_size_kb,omitempty"` + // Emit a batch once this many chunks accumulate. 0 means the server default. + BatchSize uint32 `protobuf:"varint,5,opt,name=batch_size,json=batchSize,proto3" json:"batch_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IndexRequest) Reset() { + *x = IndexRequest{} + mi := &file_indexer_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IndexRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IndexRequest) ProtoMessage() {} + +func (x *IndexRequest) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IndexRequest.ProtoReflect.Descriptor instead. +func (*IndexRequest) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{0} +} + +func (x *IndexRequest) GetRepoId() string { + if x != nil { + return x.RepoId + } + return "" +} + +func (x *IndexRequest) GetRootPath() string { + if x != nil { + return x.RootPath + } + return "" +} + +func (x *IndexRequest) GetMaxFiles() uint32 { + if x != nil { + return x.MaxFiles + } + return 0 +} + +func (x *IndexRequest) GetMaxFileSizeKb() uint32 { + if x != nil { + return x.MaxFileSizeKb + } + return 0 +} + +func (x *IndexRequest) GetBatchSize() uint32 { + if x != nil { + return x.BatchSize + } + return 0 +} + +type IndexEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Event: + // + // *IndexEvent_Progress + // *IndexEvent_Chunks + // *IndexEvent_Completed + // *IndexEvent_Failed + Event isIndexEvent_Event `protobuf_oneof:"event"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IndexEvent) Reset() { + *x = IndexEvent{} + mi := &file_indexer_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IndexEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IndexEvent) ProtoMessage() {} + +func (x *IndexEvent) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IndexEvent.ProtoReflect.Descriptor instead. +func (*IndexEvent) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{1} +} + +func (x *IndexEvent) GetEvent() isIndexEvent_Event { + if x != nil { + return x.Event + } + return nil +} + +func (x *IndexEvent) GetProgress() *Progress { + if x != nil { + if x, ok := x.Event.(*IndexEvent_Progress); ok { + return x.Progress + } + } + return nil +} + +func (x *IndexEvent) GetChunks() *ChunkBatch { + if x != nil { + if x, ok := x.Event.(*IndexEvent_Chunks); ok { + return x.Chunks + } + } + return nil +} + +func (x *IndexEvent) GetCompleted() *Completed { + if x != nil { + if x, ok := x.Event.(*IndexEvent_Completed); ok { + return x.Completed + } + } + return nil +} + +func (x *IndexEvent) GetFailed() *Failed { + if x != nil { + if x, ok := x.Event.(*IndexEvent_Failed); ok { + return x.Failed + } + } + return nil +} + +type isIndexEvent_Event interface { + isIndexEvent_Event() +} + +type IndexEvent_Progress struct { + Progress *Progress `protobuf:"bytes,1,opt,name=progress,proto3,oneof"` +} + +type IndexEvent_Chunks struct { + Chunks *ChunkBatch `protobuf:"bytes,2,opt,name=chunks,proto3,oneof"` +} + +type IndexEvent_Completed struct { + Completed *Completed `protobuf:"bytes,3,opt,name=completed,proto3,oneof"` +} + +type IndexEvent_Failed struct { + Failed *Failed `protobuf:"bytes,4,opt,name=failed,proto3,oneof"` +} + +func (*IndexEvent_Progress) isIndexEvent_Event() {} + +func (*IndexEvent_Chunks) isIndexEvent_Event() {} + +func (*IndexEvent_Completed) isIndexEvent_Event() {} + +func (*IndexEvent_Failed) isIndexEvent_Event() {} + +type Progress struct { + state protoimpl.MessageState `protogen:"open.v1"` + Stage string `protobuf:"bytes,1,opt,name=stage,proto3" json:"stage,omitempty"` // walking | parsing + CurrentPath string `protobuf:"bytes,2,opt,name=current_path,json=currentPath,proto3" json:"current_path,omitempty"` + FilesProcessed uint32 `protobuf:"varint,3,opt,name=files_processed,json=filesProcessed,proto3" json:"files_processed,omitempty"` + TotalFiles uint32 `protobuf:"varint,4,opt,name=total_files,json=totalFiles,proto3" json:"total_files,omitempty"` + Percent float64 `protobuf:"fixed64,5,opt,name=percent,proto3" json:"percent,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Progress) Reset() { + *x = Progress{} + mi := &file_indexer_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Progress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Progress) ProtoMessage() {} + +func (x *Progress) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Progress.ProtoReflect.Descriptor instead. +func (*Progress) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{2} +} + +func (x *Progress) GetStage() string { + if x != nil { + return x.Stage + } + return "" +} + +func (x *Progress) GetCurrentPath() string { + if x != nil { + return x.CurrentPath + } + return "" +} + +func (x *Progress) GetFilesProcessed() uint32 { + if x != nil { + return x.FilesProcessed + } + return 0 +} + +func (x *Progress) GetTotalFiles() uint32 { + if x != nil { + return x.TotalFiles + } + return 0 +} + +func (x *Progress) GetPercent() float64 { + if x != nil { + return x.Percent + } + return 0 +} + +// Batched rather than one chunk per message: a per-chunk stream on a large repository +// spends more time in framing than in parsing. +type ChunkBatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Chunks []*Chunk `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChunkBatch) Reset() { + *x = ChunkBatch{} + mi := &file_indexer_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkBatch) ProtoMessage() {} + +func (x *ChunkBatch) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkBatch.ProtoReflect.Descriptor instead. +func (*ChunkBatch) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{3} +} + +func (x *ChunkBatch) GetChunks() []*Chunk { + if x != nil { + return x.Chunks + } + return nil +} + +type Chunk struct { + state protoimpl.MessageState `protogen:"open.v1"` + FilePath string `protobuf:"bytes,1,opt,name=file_path,json=filePath,proto3" json:"file_path,omitempty"` // repo-relative, matching CodeFile.path on the Python side + Language string `protobuf:"bytes,2,opt,name=language,proto3" json:"language,omitempty"` + ChunkType string `protobuf:"bytes,3,opt,name=chunk_type,json=chunkType,proto3" json:"chunk_type,omitempty"` // function | class | method | module + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` + Content string `protobuf:"bytes,5,opt,name=content,proto3" json:"content,omitempty"` + StartLine uint32 `protobuf:"varint,6,opt,name=start_line,json=startLine,proto3" json:"start_line,omitempty"` + EndLine uint32 `protobuf:"varint,7,opt,name=end_line,json=endLine,proto3" json:"end_line,omitempty"` + // True when tree-sitter produced an ERROR node for this file. The Python indexer + // treats a parse error as a signal to fall back to raw indexing; without this flag + // that decision cannot cross the boundary, since tree-sitter does not raise -- it + // returns a tree containing errors. + HadParseError bool `protobuf:"varint,8,opt,name=had_parse_error,json=hadParseError,proto3" json:"had_parse_error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Chunk) Reset() { + *x = Chunk{} + mi := &file_indexer_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Chunk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Chunk) ProtoMessage() {} + +func (x *Chunk) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Chunk.ProtoReflect.Descriptor instead. +func (*Chunk) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{4} +} + +func (x *Chunk) GetFilePath() string { + if x != nil { + return x.FilePath + } + return "" +} + +func (x *Chunk) GetLanguage() string { + if x != nil { + return x.Language + } + return "" +} + +func (x *Chunk) GetChunkType() string { + if x != nil { + return x.ChunkType + } + return "" +} + +func (x *Chunk) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Chunk) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *Chunk) GetStartLine() uint32 { + if x != nil { + return x.StartLine + } + return 0 +} + +func (x *Chunk) GetEndLine() uint32 { + if x != nil { + return x.EndLine + } + return 0 +} + +func (x *Chunk) GetHadParseError() bool { + if x != nil { + return x.HadParseError + } + return false +} + +type Completed struct { + state protoimpl.MessageState `protogen:"open.v1"` + FilesWalked uint32 `protobuf:"varint,1,opt,name=files_walked,json=filesWalked,proto3" json:"files_walked,omitempty"` + FilesParsed uint32 `protobuf:"varint,2,opt,name=files_parsed,json=filesParsed,proto3" json:"files_parsed,omitempty"` + FilesSkipped uint32 `protobuf:"varint,3,opt,name=files_skipped,json=filesSkipped,proto3" json:"files_skipped,omitempty"` + ChunksEmitted uint32 `protobuf:"varint,4,opt,name=chunks_emitted,json=chunksEmitted,proto3" json:"chunks_emitted,omitempty"` + FilesWithErrors uint32 `protobuf:"varint,5,opt,name=files_with_errors,json=filesWithErrors,proto3" json:"files_with_errors,omitempty"` + DurationMs int64 `protobuf:"varint,6,opt,name=duration_ms,json=durationMs,proto3" json:"duration_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Completed) Reset() { + *x = Completed{} + mi := &file_indexer_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Completed) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Completed) ProtoMessage() {} + +func (x *Completed) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Completed.ProtoReflect.Descriptor instead. +func (*Completed) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{5} +} + +func (x *Completed) GetFilesWalked() uint32 { + if x != nil { + return x.FilesWalked + } + return 0 +} + +func (x *Completed) GetFilesParsed() uint32 { + if x != nil { + return x.FilesParsed + } + return 0 +} + +func (x *Completed) GetFilesSkipped() uint32 { + if x != nil { + return x.FilesSkipped + } + return 0 +} + +func (x *Completed) GetChunksEmitted() uint32 { + if x != nil { + return x.ChunksEmitted + } + return 0 +} + +func (x *Completed) GetFilesWithErrors() uint32 { + if x != nil { + return x.FilesWithErrors + } + return 0 +} + +func (x *Completed) GetDurationMs() int64 { + if x != nil { + return x.DurationMs + } + return 0 +} + +type Failed struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` // set when a single file caused it + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Failed) Reset() { + *x = Failed{} + mi := &file_indexer_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Failed) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Failed) ProtoMessage() {} + +func (x *Failed) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Failed.ProtoReflect.Descriptor instead. +func (*Failed) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{6} +} + +func (x *Failed) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *Failed) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +type HealthRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthRequest) Reset() { + *x = HealthRequest{} + mi := &file_indexer_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthRequest) ProtoMessage() {} + +func (x *HealthRequest) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthRequest.ProtoReflect.Descriptor instead. +func (*HealthRequest) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{7} +} + +type HealthResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ok bool `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Parsers []*ParserInfo `protobuf:"bytes,3,rep,name=parsers,proto3" json:"parsers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthResponse) Reset() { + *x = HealthResponse{} + mi := &file_indexer_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthResponse) ProtoMessage() {} + +func (x *HealthResponse) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthResponse.ProtoReflect.Descriptor instead. +func (*HealthResponse) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{8} +} + +func (x *HealthResponse) GetOk() bool { + if x != nil { + return x.Ok + } + return false +} + +func (x *HealthResponse) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *HealthResponse) GetParsers() []*ParserInfo { + if x != nil { + return x.Parsers + } + return nil +} + +type ParserInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Language string `protobuf:"bytes,1,opt,name=language,proto3" json:"language,omitempty"` + Extensions []string `protobuf:"bytes,2,rep,name=extensions,proto3" json:"extensions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ParserInfo) Reset() { + *x = ParserInfo{} + mi := &file_indexer_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ParserInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ParserInfo) ProtoMessage() {} + +func (x *ParserInfo) ProtoReflect() protoreflect.Message { + mi := &file_indexer_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ParserInfo.ProtoReflect.Descriptor instead. +func (*ParserInfo) Descriptor() ([]byte, []int) { + return file_indexer_proto_rawDescGZIP(), []int{9} +} + +func (x *ParserInfo) GetLanguage() string { + if x != nil { + return x.Language + } + return "" +} + +func (x *ParserInfo) GetExtensions() []string { + if x != nil { + return x.Extensions + } + return nil +} + +var File_indexer_proto protoreflect.FileDescriptor + +const file_indexer_proto_rawDesc = "" + + "\n" + + "\rindexer.proto\x12\x15codebaseqa.indexer.v1\"\xa9\x01\n" + + "\fIndexRequest\x12\x17\n" + + "\arepo_id\x18\x01 \x01(\tR\x06repoId\x12\x1b\n" + + "\troot_path\x18\x02 \x01(\tR\brootPath\x12\x1b\n" + + "\tmax_files\x18\x03 \x01(\rR\bmaxFiles\x12'\n" + + "\x10max_file_size_kb\x18\x04 \x01(\rR\rmaxFileSizeKb\x12\x1d\n" + + "\n" + + "batch_size\x18\x05 \x01(\rR\tbatchSize\"\x8c\x02\n" + + "\n" + + "IndexEvent\x12=\n" + + "\bprogress\x18\x01 \x01(\v2\x1f.codebaseqa.indexer.v1.ProgressH\x00R\bprogress\x12;\n" + + "\x06chunks\x18\x02 \x01(\v2!.codebaseqa.indexer.v1.ChunkBatchH\x00R\x06chunks\x12@\n" + + "\tcompleted\x18\x03 \x01(\v2 .codebaseqa.indexer.v1.CompletedH\x00R\tcompleted\x127\n" + + "\x06failed\x18\x04 \x01(\v2\x1d.codebaseqa.indexer.v1.FailedH\x00R\x06failedB\a\n" + + "\x05event\"\xa7\x01\n" + + "\bProgress\x12\x14\n" + + "\x05stage\x18\x01 \x01(\tR\x05stage\x12!\n" + + "\fcurrent_path\x18\x02 \x01(\tR\vcurrentPath\x12'\n" + + "\x0ffiles_processed\x18\x03 \x01(\rR\x0efilesProcessed\x12\x1f\n" + + "\vtotal_files\x18\x04 \x01(\rR\n" + + "totalFiles\x12\x18\n" + + "\apercent\x18\x05 \x01(\x01R\apercent\"B\n" + + "\n" + + "ChunkBatch\x124\n" + + "\x06chunks\x18\x01 \x03(\v2\x1c.codebaseqa.indexer.v1.ChunkR\x06chunks\"\xef\x01\n" + + "\x05Chunk\x12\x1b\n" + + "\tfile_path\x18\x01 \x01(\tR\bfilePath\x12\x1a\n" + + "\blanguage\x18\x02 \x01(\tR\blanguage\x12\x1d\n" + + "\n" + + "chunk_type\x18\x03 \x01(\tR\tchunkType\x12\x12\n" + + "\x04name\x18\x04 \x01(\tR\x04name\x12\x18\n" + + "\acontent\x18\x05 \x01(\tR\acontent\x12\x1d\n" + + "\n" + + "start_line\x18\x06 \x01(\rR\tstartLine\x12\x19\n" + + "\bend_line\x18\a \x01(\rR\aendLine\x12&\n" + + "\x0fhad_parse_error\x18\b \x01(\bR\rhadParseError\"\xea\x01\n" + + "\tCompleted\x12!\n" + + "\ffiles_walked\x18\x01 \x01(\rR\vfilesWalked\x12!\n" + + "\ffiles_parsed\x18\x02 \x01(\rR\vfilesParsed\x12#\n" + + "\rfiles_skipped\x18\x03 \x01(\rR\ffilesSkipped\x12%\n" + + "\x0echunks_emitted\x18\x04 \x01(\rR\rchunksEmitted\x12*\n" + + "\x11files_with_errors\x18\x05 \x01(\rR\x0ffilesWithErrors\x12\x1f\n" + + "\vduration_ms\x18\x06 \x01(\x03R\n" + + "durationMs\"6\n" + + "\x06Failed\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage\x12\x12\n" + + "\x04path\x18\x02 \x01(\tR\x04path\"\x0f\n" + + "\rHealthRequest\"w\n" + + "\x0eHealthResponse\x12\x0e\n" + + "\x02ok\x18\x01 \x01(\bR\x02ok\x12\x18\n" + + "\aversion\x18\x02 \x01(\tR\aversion\x12;\n" + + "\aparsers\x18\x03 \x03(\v2!.codebaseqa.indexer.v1.ParserInfoR\aparsers\"H\n" + + "\n" + + "ParserInfo\x12\x1a\n" + + "\blanguage\x18\x01 \x01(\tR\blanguage\x12\x1e\n" + + "\n" + + "extensions\x18\x02 \x03(\tR\n" + + "extensions2\xb7\x01\n" + + "\aIndexer\x12U\n" + + "\tIndexRepo\x12#.codebaseqa.indexer.v1.IndexRequest\x1a!.codebaseqa.indexer.v1.IndexEvent0\x01\x12U\n" + + "\x06Health\x12$.codebaseqa.indexer.v1.HealthRequest\x1a%.codebaseqa.indexer.v1.HealthResponseB codebaseqa.indexer.v1.Progress + 3, // 1: codebaseqa.indexer.v1.IndexEvent.chunks:type_name -> codebaseqa.indexer.v1.ChunkBatch + 5, // 2: codebaseqa.indexer.v1.IndexEvent.completed:type_name -> codebaseqa.indexer.v1.Completed + 6, // 3: codebaseqa.indexer.v1.IndexEvent.failed:type_name -> codebaseqa.indexer.v1.Failed + 4, // 4: codebaseqa.indexer.v1.ChunkBatch.chunks:type_name -> codebaseqa.indexer.v1.Chunk + 9, // 5: codebaseqa.indexer.v1.HealthResponse.parsers:type_name -> codebaseqa.indexer.v1.ParserInfo + 0, // 6: codebaseqa.indexer.v1.Indexer.IndexRepo:input_type -> codebaseqa.indexer.v1.IndexRequest + 7, // 7: codebaseqa.indexer.v1.Indexer.Health:input_type -> codebaseqa.indexer.v1.HealthRequest + 1, // 8: codebaseqa.indexer.v1.Indexer.IndexRepo:output_type -> codebaseqa.indexer.v1.IndexEvent + 8, // 9: codebaseqa.indexer.v1.Indexer.Health:output_type -> codebaseqa.indexer.v1.HealthResponse + 8, // [8:10] is the sub-list for method output_type + 6, // [6:8] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name +} + +func init() { file_indexer_proto_init() } +func file_indexer_proto_init() { + if File_indexer_proto != nil { + return + } + file_indexer_proto_msgTypes[1].OneofWrappers = []any{ + (*IndexEvent_Progress)(nil), + (*IndexEvent_Chunks)(nil), + (*IndexEvent_Completed)(nil), + (*IndexEvent_Failed)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_indexer_proto_rawDesc), len(file_indexer_proto_rawDesc)), + NumEnums: 0, + NumMessages: 10, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_indexer_proto_goTypes, + DependencyIndexes: file_indexer_proto_depIdxs, + MessageInfos: file_indexer_proto_msgTypes, + }.Build() + File_indexer_proto = out.File + file_indexer_proto_goTypes = nil + file_indexer_proto_depIdxs = nil +} diff --git a/services/indexer/gen/indexer_grpc.pb.go b/services/indexer/gen/indexer_grpc.pb.go new file mode 100644 index 0000000..fa477ca --- /dev/null +++ b/services/indexer/gen/indexer_grpc.pb.go @@ -0,0 +1,205 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v5.29.3 +// source: indexer.proto + +package gen + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Indexer_IndexRepo_FullMethodName = "/codebaseqa.indexer.v1.Indexer/IndexRepo" + Indexer_Health_FullMethodName = "/codebaseqa.indexer.v1.Indexer/Health" +) + +// IndexerClient is the client API for Indexer service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Repository indexing: file walk plus tree-sitter parsing, extracted from the Python API. +// +// WHY A SEPARATE SERVICE, STATED HONESTLY +// Not for parse throughput. Measured on this repository, ThreadPoolExecutor over the +// Python parser gives 1.02x (py-tree-sitter never releases the GIL) and +// ProcessPoolExecutor gives 2.84x, while only ~41% of the parse phase is actually C -- so +// a perfect native rewrite ceilings around 2.4x, below what a half-day change to the +// existing Python already achieves. +// +// The real reason is a process and failure boundary. repos.py runs indexing by spinning a +// new event loop inside an anyio threadpool thread and blocking it for minutes, in the +// same process that serves chat. This moves that CPU-bound, GIL-bound, minutes-long stage +// behind an explicit contract. +// +// Server streaming is the point of using gRPC here rather than one big response: progress +// becomes part of the contract instead of shared mutable state, which is the same bug +// class that made the SSE progress bar show only 0% or 100%. +type IndexerClient interface { + // Walk and parse a checkout, streaming progress and chunk batches as they are produced. + IndexRepo(ctx context.Context, in *IndexRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[IndexEvent], error) + // Cheap liveness probe. Returns the grammars this build actually links, which is not + // guaranteed to match the Python side -- see ParserInfo. + Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error) +} + +type indexerClient struct { + cc grpc.ClientConnInterface +} + +func NewIndexerClient(cc grpc.ClientConnInterface) IndexerClient { + return &indexerClient{cc} +} + +func (c *indexerClient) IndexRepo(ctx context.Context, in *IndexRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[IndexEvent], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Indexer_ServiceDesc.Streams[0], Indexer_IndexRepo_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[IndexRequest, IndexEvent]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Indexer_IndexRepoClient = grpc.ServerStreamingClient[IndexEvent] + +func (c *indexerClient) Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(HealthResponse) + err := c.cc.Invoke(ctx, Indexer_Health_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// IndexerServer is the server API for Indexer service. +// All implementations must embed UnimplementedIndexerServer +// for forward compatibility. +// +// Repository indexing: file walk plus tree-sitter parsing, extracted from the Python API. +// +// WHY A SEPARATE SERVICE, STATED HONESTLY +// Not for parse throughput. Measured on this repository, ThreadPoolExecutor over the +// Python parser gives 1.02x (py-tree-sitter never releases the GIL) and +// ProcessPoolExecutor gives 2.84x, while only ~41% of the parse phase is actually C -- so +// a perfect native rewrite ceilings around 2.4x, below what a half-day change to the +// existing Python already achieves. +// +// The real reason is a process and failure boundary. repos.py runs indexing by spinning a +// new event loop inside an anyio threadpool thread and blocking it for minutes, in the +// same process that serves chat. This moves that CPU-bound, GIL-bound, minutes-long stage +// behind an explicit contract. +// +// Server streaming is the point of using gRPC here rather than one big response: progress +// becomes part of the contract instead of shared mutable state, which is the same bug +// class that made the SSE progress bar show only 0% or 100%. +type IndexerServer interface { + // Walk and parse a checkout, streaming progress and chunk batches as they are produced. + IndexRepo(*IndexRequest, grpc.ServerStreamingServer[IndexEvent]) error + // Cheap liveness probe. Returns the grammars this build actually links, which is not + // guaranteed to match the Python side -- see ParserInfo. + Health(context.Context, *HealthRequest) (*HealthResponse, error) + mustEmbedUnimplementedIndexerServer() +} + +// UnimplementedIndexerServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedIndexerServer struct{} + +func (UnimplementedIndexerServer) IndexRepo(*IndexRequest, grpc.ServerStreamingServer[IndexEvent]) error { + return status.Error(codes.Unimplemented, "method IndexRepo not implemented") +} +func (UnimplementedIndexerServer) Health(context.Context, *HealthRequest) (*HealthResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Health not implemented") +} +func (UnimplementedIndexerServer) mustEmbedUnimplementedIndexerServer() {} +func (UnimplementedIndexerServer) testEmbeddedByValue() {} + +// UnsafeIndexerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to IndexerServer will +// result in compilation errors. +type UnsafeIndexerServer interface { + mustEmbedUnimplementedIndexerServer() +} + +func RegisterIndexerServer(s grpc.ServiceRegistrar, srv IndexerServer) { + // If the following call panics, it indicates UnimplementedIndexerServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Indexer_ServiceDesc, srv) +} + +func _Indexer_IndexRepo_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(IndexRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(IndexerServer).IndexRepo(m, &grpc.GenericServerStream[IndexRequest, IndexEvent]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Indexer_IndexRepoServer = grpc.ServerStreamingServer[IndexEvent] + +func _Indexer_Health_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IndexerServer).Health(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Indexer_Health_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IndexerServer).Health(ctx, req.(*HealthRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Indexer_ServiceDesc is the grpc.ServiceDesc for Indexer service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Indexer_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "codebaseqa.indexer.v1.Indexer", + HandlerType: (*IndexerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Health", + Handler: _Indexer_Health_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "IndexRepo", + Handler: _Indexer_IndexRepo_Handler, + ServerStreams: true, + }, + }, + Metadata: "indexer.proto", +} diff --git a/services/indexer/go.mod b/services/indexer/go.mod new file mode 100644 index 0000000..2e18d1b --- /dev/null +++ b/services/indexer/go.mod @@ -0,0 +1,21 @@ +module github.com/ShreeBohara/codebaseqa/services/indexer + +go 1.25.0 + +require ( + github.com/tree-sitter/go-tree-sitter v0.25.0 + github.com/tree-sitter/tree-sitter-go v0.25.0 + github.com/tree-sitter/tree-sitter-javascript v0.25.0 + github.com/tree-sitter/tree-sitter-python v0.25.0 + github.com/tree-sitter/tree-sitter-typescript v0.23.2 + google.golang.org/grpc v1.83.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + github.com/mattn/go-pointer v0.0.1 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect +) diff --git a/services/indexer/go.sum b/services/indexer/go.sum new file mode 100644 index 0000000..8e61545 --- /dev/null +++ b/services/indexer/go.sum @@ -0,0 +1,76 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tree-sitter/go-tree-sitter v0.25.0 h1:sx6kcg8raRFCvc9BnXglke6axya12krCJF5xJ2sftRU= +github.com/tree-sitter/go-tree-sitter v0.25.0/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU= +github.com/tree-sitter/tree-sitter-c v0.23.4 h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c= +github.com/tree-sitter/tree-sitter-c v0.23.4/go.mod h1:MkI5dOiIpeN94LNjeCp8ljXN/953JCwAby4bClMr6bw= +github.com/tree-sitter/tree-sitter-cpp v0.23.4 h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w= +github.com/tree-sitter/tree-sitter-cpp v0.23.4/go.mod h1:doqNW64BriC7WBCQ1klf0KmJpdEvfxyXtoEybnBo6v8= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2 h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2/go.mod h1:HNPOhN0qF3hWluYLdxWs5WbzP/iE4aaRVPMsdxuzIaQ= +github.com/tree-sitter/tree-sitter-go v0.25.0 h1:cEB0Q3LHgZtS+ECHx9wcP7AwzoOddJFQCVmytX42cVU= +github.com/tree-sitter/tree-sitter-go v0.25.0/go.mod h1:Jrx8QqYN0v7npv1fJRH1AznddllYiCMUChtVjxPK040= +github.com/tree-sitter/tree-sitter-html v0.23.2 h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc= +github.com/tree-sitter/tree-sitter-html v0.23.2/go.mod h1:gpUv/dG3Xl/eebqgeYeFMt+JLOY9cgFinb/Nw08a9og= +github.com/tree-sitter/tree-sitter-java v0.23.5 h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig= +github.com/tree-sitter/tree-sitter-java v0.23.5/go.mod h1:NRKlI8+EznxA7t1Yt3xtraPk1Wzqh3GAIC46wxvc320= +github.com/tree-sitter/tree-sitter-javascript v0.25.0 h1:ZkWETb66/w8cc13yhfnNuHOLDQWl3BnKlH6f9AdR88c= +github.com/tree-sitter/tree-sitter-javascript v0.25.0/go.mod h1:lmGD1EJdCA+v0S1u2fFgepMg/opzSg/4pgFym2FPGAs= +github.com/tree-sitter/tree-sitter-json v0.24.8 h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ= +github.com/tree-sitter/tree-sitter-json v0.24.8/go.mod h1:F351KK0KGvCaYbZ5zxwx/gWWvZhIDl0eMtn+1r+gQbo= +github.com/tree-sitter/tree-sitter-php v0.23.11 h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8= +github.com/tree-sitter/tree-sitter-php v0.23.11/go.mod h1:T/kbfi+UcCywQfUNAJnGTN/fMSUjnwPXA8k4yoIks74= +github.com/tree-sitter/tree-sitter-python v0.25.0 h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0= +github.com/tree-sitter/tree-sitter-python v0.25.0/go.mod h1:cpdthSy/Yoa28aJFBscFHlGiU+cnSiSh1kuDVtI8YeM= +github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo= +github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA= +github.com/tree-sitter/tree-sitter-rust v0.23.2 h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90= +github.com/tree-sitter/tree-sitter-rust v0.23.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI= +github.com/tree-sitter/tree-sitter-typescript v0.23.2 h1:/Odvphn18PniVixb9e97X0DbNVsU6Qocv9mfkyzdXwU= +github.com/tree-sitter/tree-sitter-typescript v0.23.2/go.mod h1:zjzMXT/Ulffel2xfOcAkQQkiAkmgnbtPGlFQw/5X4xA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= +google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/indexer/internal/parse/parse.go b/services/indexer/internal/parse/parse.go new file mode 100644 index 0000000..60ae5a4 --- /dev/null +++ b/services/indexer/internal/parse/parse.go @@ -0,0 +1,251 @@ +// Package parse extracts semantic chunks with tree-sitter. +// +// Uses the OFFICIAL tree-sitter/go-tree-sitter binding. Not smacker/go-tree-sitter, which +// outranks it in search results and has roughly twice the stars but was last pushed +// 2024-08-27 with 42 open issues and no deprecation notice -- it is the default wrong +// choice here. +// +// Two correctness details carried over from fixing the same bugs in the Python parser: +// +// 1. .tsx uses LanguageTSX(), not LanguageTypescript(). The plain TypeScript grammar +// cannot parse JSX and returns a tree full of ERROR nodes for every .tsx file. +// 2. Chunk text comes from node.Utf8Text over the source bytes. tree-sitter offsets are +// BYTE offsets; slicing a decoded string with them corrupts every chunk after the +// first multi-byte character. +package parse + +import ( + "os" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" + golang "github.com/tree-sitter/tree-sitter-go/bindings/go" + javascript "github.com/tree-sitter/tree-sitter-javascript/bindings/go" + python "github.com/tree-sitter/tree-sitter-python/bindings/go" + typescript "github.com/tree-sitter/tree-sitter-typescript/bindings/go" +) + +type Chunk struct { + FilePath string + Language string + ChunkType string + Name string + Content string + StartLine uint32 + EndLine uint32 + HadParseError bool +} + +type langConfig struct { + name string + language *tree_sitter.Language + functionTypes map[string]struct{} + classTypes map[string]struct{} + nameTypes map[string]struct{} +} + +func set(items ...string) map[string]struct{} { + m := make(map[string]struct{}, len(items)) + for _, i := range items { + m[i] = struct{}{} + } + return m +} + +// Grammar set is deliberately narrower than the Python side's nine. Each grammar is cgo +// with its own generated parser.c, so every addition costs build time and image size; +// these four cover the languages this project is actually indexed against. Health() +// reports what is linked so the caller never has to assume parity. +var configs = map[string]langConfig{ + "python": { + name: "python", + language: tree_sitter.NewLanguage(python.Language()), + functionTypes: set("function_definition"), + classTypes: set("class_definition"), + nameTypes: set("identifier"), + }, + "javascript": { + name: "javascript", + language: tree_sitter.NewLanguage(javascript.Language()), + functionTypes: set("function_declaration", "method_definition", "arrow_function", "function_expression"), + classTypes: set("class_declaration"), + nameTypes: set("identifier", "property_identifier"), + }, + "typescript": { + name: "typescript", + language: tree_sitter.NewLanguage(typescript.LanguageTypescript()), + functionTypes: set("function_declaration", "method_definition", "arrow_function", "function_signature"), + classTypes: set("class_declaration", "interface_declaration"), + nameTypes: set("identifier", "property_identifier", "type_identifier"), + }, + // Separate entry precisely because LanguageTypescript() cannot parse JSX. + "tsx": { + name: "tsx", + language: tree_sitter.NewLanguage(typescript.LanguageTSX()), + functionTypes: set("function_declaration", "method_definition", "arrow_function", "function_signature"), + classTypes: set("class_declaration", "interface_declaration"), + nameTypes: set("identifier", "property_identifier", "type_identifier"), + }, + "go": { + name: "go", + language: tree_sitter.NewLanguage(golang.Language()), + functionTypes: set("function_declaration", "method_declaration"), + classTypes: set("type_declaration"), + nameTypes: set("identifier", "type_identifier", "field_identifier"), + }, +} + +var extToLang = map[string]string{ + ".py": "python", + ".js": "javascript", ".jsx": "javascript", + ".ts": "typescript", + ".tsx": "tsx", // never "typescript" + ".go": "go", +} + +// LanguageFor reports the grammar for an extension, and whether one exists. +func LanguageFor(ext string) (string, bool) { + l, ok := extToLang[ext] + return l, ok +} + +// Linked returns the grammars this build links, for Health(). +func Linked() map[string][]string { + out := map[string][]string{} + for ext, lang := range extToLang { + out[lang] = append(out[lang], ext) + } + return out +} + +// File parses one file and returns its chunks. +// +// A tree containing ERROR nodes is reported via HadParseError rather than swallowed: +// tree-sitter does not fail on a syntax error, it returns a partial tree, so a caller +// that only watches for errors would silently index garbage. The Python side uses the +// same signal to fall back to raw indexing. +func File(absPath, relPath string) ([]Chunk, error) { + ext := lowerExt(relPath) + langName, ok := LanguageFor(ext) + if !ok { + return nil, nil + } + cfg := configs[langName] + + source, err := os.ReadFile(absPath) + if err != nil { + return nil, err + } + + parser := tree_sitter.NewParser() + defer parser.Close() + if err := parser.SetLanguage(cfg.language); err != nil { + return nil, err + } + + tree := parser.Parse(source, nil) + if tree == nil { + return nil, nil + } + defer tree.Close() + + root := tree.RootNode() + hadError := root.HasError() + + var chunks []Chunk + var visit func(n *tree_sitter.Node, inClass bool) + visit = func(n *tree_sitter.Node, inClass bool) { + kind := n.Kind() + _, isClass := cfg.classTypes[kind] + _, isFunc := cfg.functionTypes[kind] + + switch { + case isClass: + chunks = append(chunks, chunkFrom(n, source, relPath, cfg, "class", hadError)) + case isFunc: + kindLabel := "function" + if inClass { + kindLabel = "method" + } + chunks = append(chunks, chunkFrom(n, source, relPath, cfg, kindLabel, hadError)) + } + + for i := uint(0); i < n.ChildCount(); i++ { + visit(n.Child(i), inClass || isClass) + } + } + visit(root, false) + + // Mirrors the Python fallback: a file with no extractable declarations still carries + // meaning, so emit it as a module-level chunk rather than dropping it. + if len(chunks) == 0 && len(source) > 0 { + chunks = append(chunks, Chunk{ + FilePath: relPath, + Language: cfg.name, + ChunkType: "module", + Content: truncate(string(source), 4000), + StartLine: 1, + EndLine: uint32(root.EndPosition().Row) + 1, + HadParseError: hadError, + }) + } + return chunks, nil +} + +func chunkFrom( + n *tree_sitter.Node, source []byte, relPath string, cfg langConfig, + chunkType string, hadError bool, +) Chunk { + return Chunk{ + FilePath: relPath, + Language: cfg.name, + ChunkType: chunkType, + Name: nameOf(n, source, cfg), + // Utf8Text slices the SOURCE BYTES, which is what tree-sitter offsets index. + Content: n.Utf8Text(source), + StartLine: uint32(n.StartPosition().Row) + 1, + EndLine: uint32(n.EndPosition().Row) + 1, + HadParseError: hadError, + } +} + +func nameOf(n *tree_sitter.Node, source []byte, cfg langConfig) string { + if named := n.ChildByFieldName("name"); named != nil { + return named.Utf8Text(source) + } + for i := uint(0); i < n.ChildCount(); i++ { + child := n.Child(i) + if _, ok := cfg.nameTypes[child.Kind()]; ok { + return child.Utf8Text(source) + } + } + return "" +} + +func lowerExt(path string) string { + for i := len(path) - 1; i >= 0; i-- { + if path[i] == '.' { + return lower(path[i:]) + } + if path[i] == '/' { + break + } + } + return "" +} + +func lower(s string) string { + b := []byte(s) + for i := range b { + if b[i] >= 'A' && b[i] <= 'Z' { + b[i] += 32 + } + } + return string(b) +} + +func truncate(s string, max int) string { + if len(s) <= max { + return s + } + return s[:max] + "\n... [truncated]" +} diff --git a/services/indexer/internal/walk/walk.go b/services/indexer/internal/walk/walk.go new file mode 100644 index 0000000..6a179b0 --- /dev/null +++ b/services/indexer/internal/walk/walk.go @@ -0,0 +1,124 @@ +// Package walk finds indexable files in a checkout. +// +// Ported from IndexingService._find_files in the Python API, with the same skip +// directories, extension set and size cap so both sides agree on what "indexable" means. +// One deliberate difference: the Python version's max-files cap did not work. Its `break` +// left only the inner filename loop, so os.walk continued into the next directory and kept +// appending -- the cap leaked by up to one directory's worth each time. Here the limit is +// enforced with fs.SkipAll, which actually stops the walk. +package walk + +import ( + "io/fs" + "os" + "path/filepath" + "strings" +) + +// Directories never worth indexing. Mirrors SKIP_PATTERNS in indexing_service.py. +var skipDirs = map[string]struct{}{ + "node_modules": {}, "__pycache__": {}, ".git": {}, ".venv": {}, "venv": {}, + "dist": {}, "build": {}, ".next": {}, "coverage": {}, ".pytest_cache": {}, + "vendor": {}, "target": {}, ".idea": {}, ".vscode": {}, +} + +// Mirrors INDEXED_EXTENSIONS. +var indexedExts = map[string]struct{}{ + ".py": {}, ".js": {}, ".jsx": {}, ".ts": {}, ".tsx": {}, + ".java": {}, ".go": {}, ".rs": {}, ".c": {}, ".cpp": {}, ".h": {}, ".cc": {}, + ".cxx": {}, ".hpp": {}, ".hh": {}, ".hxx": {}, ".ipp": {}, ".tpp": {}, + ".cs": {}, ".csx": {}, + ".rb": {}, ".rake": {}, ".gemspec": {}, ".php": {}, ".swift": {}, ".kt": {}, + ".erb": {}, + ".md": {}, ".json": {}, +} + +// Mirrors INDEXED_FILENAMES: extensionless files worth indexing. +var indexedNames = map[string]struct{}{ + "gemfile": {}, "rakefile": {}, "config.ru": {}, +} + +type Result struct { + // Paths relative to root, so they match CodeFile.path on the Python side. + Paths []string + Skipped int + // True when MaxFiles stopped the walk early, so the caller can report a partial + // index rather than implying the repository was fully covered. + Truncated bool +} + +type Options struct { + MaxFiles int + MaxFileSizeKB int +} + +// Find returns indexable files under root. +func Find(root string, opts Options) (Result, error) { + if opts.MaxFiles <= 0 { + opts.MaxFiles = 5000 + } + if opts.MaxFileSizeKB <= 0 { + opts.MaxFileSizeKB = 500 + } + maxBytes := int64(opts.MaxFileSizeKB) * 1024 + + res := Result{Paths: make([]string, 0, 256)} + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // An unreadable directory is not fatal: skip it and index the rest, matching + // the Python behaviour of swallowing per-entry OSError. + if d != nil && d.IsDir() { + return fs.SkipDir + } + return nil + } + + if d.IsDir() { + if _, skip := skipDirs[d.Name()]; skip { + return fs.SkipDir + } + return nil + } + + if !isIndexable(d.Name()) { + return nil + } + + info, statErr := d.Info() + if statErr != nil { + res.Skipped++ + return nil + } + if info.Size() > maxBytes { + res.Skipped++ + return nil + } + + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + res.Skipped++ + return nil + } + res.Paths = append(res.Paths, filepath.ToSlash(rel)) + + if len(res.Paths) >= opts.MaxFiles { + res.Truncated = true + return fs.SkipAll // actually stops, unlike the Python inner-loop break + } + return nil + }) + if err != nil && !os.IsNotExist(err) { + return res, err + } + return res, nil +} + +func isIndexable(name string) bool { + lower := strings.ToLower(name) + if _, ok := indexedNames[lower]; ok { + return true + } + _, ok := indexedExts[strings.ToLower(filepath.Ext(name))] + return ok +} diff --git a/services/indexer/main.go b/services/indexer/main.go new file mode 100644 index 0000000..515e1eb --- /dev/null +++ b/services/indexer/main.go @@ -0,0 +1,221 @@ +// Command indexer serves the walk-and-parse stage over gRPC. +// +// See proto/indexer.proto for why this is a separate process. In short: not for parse +// speed (measured, a native rewrite ceilings around 2.4x while ProcessPoolExecutor on the +// existing Python already gives 2.84x), but to move a CPU-bound, GIL-bound, minutes-long +// stage out of the process that serves chat, behind a typed streaming contract. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "os" + "path/filepath" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" + + "github.com/ShreeBohara/codebaseqa/services/indexer/gen" + "github.com/ShreeBohara/codebaseqa/services/indexer/internal/parse" + "github.com/ShreeBohara/codebaseqa/services/indexer/internal/walk" +) + +const version = "0.1.0" + +// Default chunks per ChunkBatch. Batching matters: one message per chunk spends more time +// in framing than in parsing on a large repository. +const defaultBatchSize = 200 + +// Progress is emitted at most this often. Without throttling a fast walk floods the +// stream with one event per file, which is the noise the Python SSE endpoint used to send +// every second. +const progressEvery = 25 + +type server struct { + gen.UnimplementedIndexerServer +} + +func (s *server) Health(_ context.Context, _ *gen.HealthRequest) (*gen.HealthResponse, error) { + var parsers []*gen.ParserInfo + for lang, exts := range parse.Linked() { + parsers = append(parsers, &gen.ParserInfo{Language: lang, Extensions: exts}) + } + return &gen.HealthResponse{Ok: true, Version: version, Parsers: parsers}, nil +} + +func (s *server) IndexRepo(req *gen.IndexRequest, stream gen.Indexer_IndexRepoServer) error { + started := time.Now() + + root := req.GetRootPath() + if root == "" { + return sendFailed(stream, "root_path is required", "") + } + info, err := os.Stat(root) + if err != nil || !info.IsDir() { + // A missing checkout is the single most likely caller mistake, so it gets a + // specific message rather than a generic failure. + return sendFailed(stream, fmt.Sprintf("root_path is not a readable directory: %s", root), root) + } + + batchSize := int(req.GetBatchSize()) + if batchSize <= 0 { + batchSize = defaultBatchSize + } + + if err := send(stream, &gen.IndexEvent{ + Event: &gen.IndexEvent_Progress{Progress: &gen.Progress{Stage: "walking", Percent: 0}}, + }); err != nil { + return err + } + + found, err := walk.Find(root, walk.Options{ + MaxFiles: int(req.GetMaxFiles()), + MaxFileSizeKB: int(req.GetMaxFileSizeKb()), + }) + if err != nil { + return sendFailed(stream, fmt.Sprintf("walk failed: %v", err), root) + } + + total := len(found.Paths) + if found.Truncated { + // Say so rather than letting the caller assume full coverage. + log.Printf("walk truncated at max_files=%d for %s", total, root) + } + + var ( + batch []*gen.Chunk + parsed int + emitted int + withErrors int + skippedParses int + ) + + flush := func() error { + if len(batch) == 0 { + return nil + } + if err := send(stream, &gen.IndexEvent{ + Event: &gen.IndexEvent_Chunks{Chunks: &gen.ChunkBatch{Chunks: batch}}, + }); err != nil { + return err + } + emitted += len(batch) + batch = batch[:0] + return nil + } + + for i, rel := range found.Paths { + // Honour client cancellation: without this, a cancelled request keeps parsing a + // whole repository for a stream nobody is reading. + if err := stream.Context().Err(); err != nil { + return err + } + + chunks, perr := parse.File(filepath.Join(root, rel), rel) + if perr != nil { + // One unreadable or unparseable file must not fail the whole index; the + // Python caller falls back to raw indexing for these. + skippedParses++ + continue + } + if len(chunks) > 0 { + parsed++ + if chunks[0].HadParseError { + withErrors++ + } + } + for _, c := range chunks { + batch = append(batch, &gen.Chunk{ + FilePath: c.FilePath, Language: c.Language, ChunkType: c.ChunkType, + Name: c.Name, Content: c.Content, + StartLine: c.StartLine, EndLine: c.EndLine, + HadParseError: c.HadParseError, + }) + if len(batch) >= batchSize { + if err := flush(); err != nil { + return err + } + } + } + + if i%progressEvery == 0 || i == total-1 { + if err := send(stream, &gen.IndexEvent{ + Event: &gen.IndexEvent_Progress{Progress: &gen.Progress{ + Stage: "parsing", CurrentPath: rel, + FilesProcessed: uint32(i + 1), TotalFiles: uint32(total), + Percent: float64(i+1) / float64(max(total, 1)) * 100, + }}, + }); err != nil { + return err + } + } + } + + if err := flush(); err != nil { + return err + } + + return send(stream, &gen.IndexEvent{ + Event: &gen.IndexEvent_Completed{Completed: &gen.Completed{ + FilesWalked: uint32(total), + FilesParsed: uint32(parsed), + FilesSkipped: uint32(found.Skipped + skippedParses), + ChunksEmitted: uint32(emitted), + FilesWithErrors: uint32(withErrors), + DurationMs: time.Since(started).Milliseconds(), + }}, + }) +} + +func send(stream gen.Indexer_IndexRepoServer, ev *gen.IndexEvent) error { + return stream.Send(ev) +} + +func sendFailed(stream gen.Indexer_IndexRepoServer, msg, path string) error { + // Delivered as a stream event rather than a gRPC error status so the caller sees it + // in the same channel as progress, and a partially useful stream still terminates + // cleanly. + return send(stream, &gen.IndexEvent{ + Event: &gen.IndexEvent_Failed{Failed: &gen.Failed{Message: msg, Path: path}}, + }) +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func main() { + addr := flag.String("addr", ":50051", "listen address") + flag.Parse() + + lis, err := net.Listen("tcp", *addr) + if err != nil { + log.Fatalf("listen %s: %v", *addr, err) + } + + grpcServer := grpc.NewServer() + gen.RegisterIndexerServer(grpcServer, &server{}) + + // Standard grpc health service, so a Kubernetes probe can use grpc_health_probe + // rather than needing an HTTP sidecar. + hs := health.NewServer() + hs.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthpb.RegisterHealthServer(grpcServer, hs) + + // Reflection so grpcurl works against a running instance without the .proto. + reflection.Register(grpcServer) + + log.Printf("indexer %s listening on %s", version, *addr) + if err := grpcServer.Serve(lis); err != nil { + log.Fatalf("serve: %v", err) + } +} diff --git a/services/indexer/proto/indexer.proto b/services/indexer/proto/indexer.proto new file mode 100644 index 0000000..fc6a144 --- /dev/null +++ b/services/indexer/proto/indexer.proto @@ -0,0 +1,110 @@ +syntax = "proto3"; + +package codebaseqa.indexer.v1; + +option go_package = "github.com/ShreeBohara/codebaseqa/services/indexer/gen;gen"; + +// Repository indexing: file walk plus tree-sitter parsing, extracted from the Python API. +// +// WHY A SEPARATE SERVICE, STATED HONESTLY +// Not for parse throughput. Measured on this repository, ThreadPoolExecutor over the +// Python parser gives 1.02x (py-tree-sitter never releases the GIL) and +// ProcessPoolExecutor gives 2.84x, while only ~41% of the parse phase is actually C -- so +// a perfect native rewrite ceilings around 2.4x, below what a half-day change to the +// existing Python already achieves. +// +// The real reason is a process and failure boundary. repos.py runs indexing by spinning a +// new event loop inside an anyio threadpool thread and blocking it for minutes, in the +// same process that serves chat. This moves that CPU-bound, GIL-bound, minutes-long stage +// behind an explicit contract. +// +// Server streaming is the point of using gRPC here rather than one big response: progress +// becomes part of the contract instead of shared mutable state, which is the same bug +// class that made the SSE progress bar show only 0% or 100%. +service Indexer { + // Walk and parse a checkout, streaming progress and chunk batches as they are produced. + rpc IndexRepo(IndexRequest) returns (stream IndexEvent); + + // Cheap liveness probe. Returns the grammars this build actually links, which is not + // guaranteed to match the Python side -- see ParserInfo. + rpc Health(HealthRequest) returns (HealthResponse); +} + +message IndexRequest { + string repo_id = 1; + // Absolute path to an existing checkout. The service does NOT clone: the API already + // owns credentials and the clone lives on a volume attached to one machine, so cloning + // here would duplicate auth handling and force the two to share a filesystem for a + // reason other than reading it. + string root_path = 2; + // 0 means "use the server default" so the API can defer rather than restate policy. + uint32 max_files = 3; + uint32 max_file_size_kb = 4; + // Emit a batch once this many chunks accumulate. 0 means the server default. + uint32 batch_size = 5; +} + +message IndexEvent { + oneof event { + Progress progress = 1; + ChunkBatch chunks = 2; + Completed completed = 3; + Failed failed = 4; + } +} + +message Progress { + string stage = 1; // walking | parsing + string current_path = 2; + uint32 files_processed = 3; + uint32 total_files = 4; + double percent = 5; +} + +// Batched rather than one chunk per message: a per-chunk stream on a large repository +// spends more time in framing than in parsing. +message ChunkBatch { + repeated Chunk chunks = 1; +} + +message Chunk { + string file_path = 1; // repo-relative, matching CodeFile.path on the Python side + string language = 2; + string chunk_type = 3; // function | class | method | module + string name = 4; + string content = 5; + uint32 start_line = 6; + uint32 end_line = 7; + // True when tree-sitter produced an ERROR node for this file. The Python indexer + // treats a parse error as a signal to fall back to raw indexing; without this flag + // that decision cannot cross the boundary, since tree-sitter does not raise -- it + // returns a tree containing errors. + bool had_parse_error = 8; +} + +message Completed { + uint32 files_walked = 1; + uint32 files_parsed = 2; + uint32 files_skipped = 3; + uint32 chunks_emitted = 4; + uint32 files_with_errors = 5; + int64 duration_ms = 6; +} + +message Failed { + string message = 1; + string path = 2; // set when a single file caused it +} + +message HealthRequest {} + +message HealthResponse { + bool ok = 1; + string version = 2; + repeated ParserInfo parsers = 3; +} + +message ParserInfo { + string language = 1; + repeated string extensions = 2; +}