From 7634904c92c6f8a06686d25892a4266c6ccb7ba7 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 15 Aug 2026 14:45:28 +0700 Subject: [PATCH 1/9] Implement MCP server for GA --- Cargo.lock | 675 +++++++++++++++++- Cargo.toml | 5 +- README.md | 18 +- crates/codegraph-core/src/lib.rs | 2 + crates/codegraph-core/src/route.rs | 60 ++ crates/codegraph-extract/Cargo.toml | 4 + crates/codegraph-graph/Cargo.toml | 4 + crates/codegraph-graph/src/lib.rs | 52 +- crates/codegraph-graph/src/radix.rs | 55 +- crates/codegraph-graph/src/search.rs | 2 +- crates/codegraph-graph/src/storage.rs | 6 + crates/codegraph-graph/src/storage/mysql.rs | 247 +++++++ .../codegraph-graph/src/storage/postgres.rs | 259 +++++++ crates/codegraph-mcp/Cargo.toml | 8 +- crates/codegraph-mcp/src/http.rs | 145 +++- crates/codegraph-mcp/src/lib.rs | 6 +- crates/codegraph/Cargo.toml | 2 +- crates/codegraph/src/main.rs | 87 ++- sql/README.md | 140 ++++ sql/mysql/001-initial-schema.sql | 239 +++++++ sql/mysql/002-add-repos-registry.sql | 42 ++ sql/postgres/001-initial-schema.sql | 248 +++++++ sql/postgres/002-add-repos-registry.sql | 39 + 23 files changed, 2247 insertions(+), 98 deletions(-) create mode 100644 crates/codegraph-core/src/route.rs create mode 100644 crates/codegraph-graph/src/storage/mysql.rs create mode 100644 crates/codegraph-graph/src/storage/postgres.rs create mode 100644 sql/README.md create mode 100644 sql/mysql/001-initial-schema.sql create mode 100644 sql/mysql/002-add-repos-registry.sql create mode 100644 sql/postgres/001-initial-schema.sql create mode 100644 sql/postgres/002-add-repos-registry.sql diff --git a/Cargo.lock b/Cargo.lock index 01248d3d3..63e185ca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,12 +154,70 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base64" version = "0.22.1" @@ -172,6 +230,12 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bincode" version = "1.3.3" @@ -192,6 +256,9 @@ name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +dependencies = [ + "serde_core", +] [[package]] name = "block-buffer" @@ -272,6 +339,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -431,6 +509,7 @@ dependencies = [ "camino", "codegraph-core", "codegraph-graph", + "getrandom 0.2.17", "ignore", "indicatif", "rayon", @@ -438,6 +517,7 @@ dependencies = [ "tempfile", "tokio", "toml", + "toml_edit", "tracing", "tree-sitter", "tree-sitter-c", @@ -504,6 +584,7 @@ name = "codegraph-mcp" version = "1.2.0" dependencies = [ "anyhow", + "axum", "camino", "codegraph-api", "codegraph-context", @@ -516,6 +597,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tower", "tracing", ] @@ -639,6 +721,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-random" version = "0.1.18" @@ -674,6 +762,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "cranelift-bforest" version = "0.116.1" @@ -957,6 +1054,17 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "digest" version = "0.10.7" @@ -964,7 +1072,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -1048,6 +1158,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -1287,6 +1408,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -1394,6 +1516,113 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1687,6 +1916,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.9", +] [[package]] name = "leb128fmt" @@ -1700,13 +1932,22 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ + "bitflags 2.11.1", "libc", + "plain", + "redox_syscall 0.7.5", ] [[package]] @@ -1788,12 +2029,34 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "mio" version = "1.2.0" @@ -1887,6 +2150,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.7", + "smallvec", + "zeroize", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1896,6 +2175,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1903,6 +2192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1956,7 +2246,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -1967,6 +2257,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1979,12 +2278,39 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "plotters" version = "0.3.7" @@ -2028,6 +2354,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -2068,6 +2403,53 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rayon" version = "1.12.0" @@ -2122,6 +2504,15 @@ dependencies = [ "bitflags 2.11.1", ] +[[package]] +name = "redox_syscall" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" +dependencies = [ + "bitflags 2.11.1", +] + [[package]] name = "redox_users" version = "0.4.6" @@ -2243,18 +2634,27 @@ version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8dddc5b1924b9a59fba420166160ca2c4663a4e01803e52eda33070f56d63c8" dependencies = [ + "async-trait", "base64 0.23.1", + "bytes", "chrono", "futures", + "http", + "http-body", + "http-body-util", "pastey", "pin-project-lite", + "rand 0.10.2", "rmcp-macros", "schemars", "serde", "serde_json", + "sse-stream", "thiserror 2.0.18", "tokio", + "tokio-stream", "tokio-util", + "tower-service", "tracing", "uuid", ] @@ -2272,6 +2672,26 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rusqlite" version = "0.32.1" @@ -2419,6 +2839,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -2440,6 +2871,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -2453,7 +2895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2472,6 +2914,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "slab" version = "0.4.12" @@ -2483,6 +2935,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "smartstring" @@ -2520,6 +2975,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlx" version = "0.8.6" @@ -2528,6 +2993,8 @@ checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" dependencies = [ "sqlx-core", "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", "sqlx-sqlite", ] @@ -2555,6 +3022,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", + "serde_json", "sha2", "smallvec", "thiserror 2.0.18", @@ -2594,12 +3062,93 @@ dependencies = [ "serde_json", "sha2", "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", "sqlx-sqlite", "syn 2.0.117", "tokio", "url", ] +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.11.1", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.7", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.11.1", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.7", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami", +] + [[package]] name = "sqlx-sqlite" version = "0.8.6" @@ -2624,6 +3173,19 @@ dependencies = [ "url", ] +[[package]] +name = "sse-stream" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" +dependencies = [ + "bytes", + "futures-util", + "http-body", + "http-body-util", + "pin-project-lite", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2652,12 +3214,29 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -2680,6 +3259,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.2" @@ -2804,6 +3389,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -2895,6 +3495,34 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -3123,12 +3751,33 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-width" version = "0.2.2" @@ -3234,6 +3883,12 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -3357,6 +4012,16 @@ dependencies = [ "winsafe", ] +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi-util" version = "0.1.11" @@ -3841,6 +4506,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index 6a2121536..25597bfe6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,10 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } # storage rusqlite = { version = "0.32", features = ["bundled", "backup"] } redis = { version = "1.0", features = ["tokio-comp"] } -sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "sqlite"] } +# postgres/mysql drivers cho backend RDBMS sharded (feature `postgres`/`mysql` +# trên codegraph-graph gate module `storage/rdbms.rs`; sqlx enable như sqlite — +# feature unification khiến driver thêm là additive cho mọi consumer). +sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "sqlite", "postgres", "mysql"] } # embedded memory-mapped KV (bundled C — no system lib needed) lmdb-rkv = "0.14" diff --git a/README.md b/README.md index 022517d96..03c14db1f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Agents that consult the semantic graph instead of grepping the filesystem make * - **Fast.** Full re-index a 139-file project in ~190 ms (release, parallel rayon). - **Local.** Index lives in `.codegraph/db.sqlite` next to your code. Nothing leaves the machine. - **Full re-index always.** No incremental sync — watcher debounces and re-indexes completely (simpler, no stale state). -- **Multi-agent.** One binary serves any MCP client (Claude Code, Cursor, Codex, opencode, Hermes, Antigravity) over stdio — the agent binds the workspace with `codegraph_init` and drives everything through tools. +- **Multi-agent.** One binary serves any MCP client (Claude Code, Cursor, Codex, opencode, Hermes, Antigravity) over stdio or Streamable HTTP (`--http`) — the agent binds the workspace with `codegraph_init` and drives everything through tools. - **30 MCP tools** including `codegraph_flow` (call chain), `codegraph_search_flow` (pattern search), `codegraph_references` (library call consumers), `codegraph_diff` (MR impact draft), and a behavior sandbox (`codegraph_sandbox`). ## Install @@ -94,8 +94,12 @@ cargo install --git https://github.com/Cleboost/codegraph-rs codegraph cd ~/code/my-project codegraph init -# 2. Serve it to your agent (Claude Code, Cursor, ...) over MCP +# 2. Serve it to your agent (Claude Code, Cursor, ...) over MCP (stdio) codegraph serve --mcp + +# ... or over Streamable HTTP (SSE), e.g. for a remote client / Docker container +codegraph serve --mcp --http --addr 0.0.0.0:8123 +# point the client at: http://:8123/mcp → {"type": "http", "url": "http://:8123/mcp"} ``` The agent then binds the workspace with `codegraph_init {"path": ...}` and gets @@ -104,6 +108,13 @@ tools like `codegraph_search`, `codegraph_symbol`, `codegraph_callers`, `codegraph_context` — all querying is done **over MCP**, not via CLI commands. The file watcher debounces changes and triggers full re-indexes while you edit. +Over HTTP each connection (`mcp-session-id`) gets its own fresh server session +— the agent binds the workspace root with `codegraph_init` inside that +connection; nothing is shared between connections but the process. rmcp's +`allowed_hosts` check blocks foreign `Host` headers (DNS-rebinding protection): +loopback hosts pass by default; for LAN access pass `--allow-host ` +(repeatable) or `--allow-any-host` on a trusted network. + ## CLI reference The CLI is deliberately minimal — it only manages the workspace lifecycle and @@ -114,6 +125,7 @@ runs the MCP server. All reading/interacting goes through MCP tools. | `codegraph init [--no-index]` | Create `.codegraph/` and full re-index (skip with `--no-index`) | | `codegraph deinit` | Remove `.codegraph/` | | `codegraph serve --mcp` | Run as MCP server over stdio (used by agents) | +| `codegraph serve --mcp --http` | Run as MCP server over Streamable HTTP (SSE); `--addr` (default `0.0.0.0:8123`), `--allow-host ` (repeatable, LAN), `--allow-any-host` | Global flag `--path ` overrides the workspace root. @@ -187,7 +199,7 @@ crates/ codegraph-graph/ GraphIndex (semgraph): registry + 2 engines (chain Search + name Search) + sqlite storage codegraph-context/ Markdown/JSON context formatter (symbol + callers + callees + source) codegraph-api/ GraphApi wrapper on SharedGraphIndex (async query surface) - codegraph-mcp/ MCP server on the rmcp SDK (stdio) + 30-tool dispatch, session-driven + codegraph-mcp/ MCP server on the rmcp SDK (stdio + Streamable HTTP) + 30-tool dispatch, session-driven codegraph-installer/ Agent config targets (Claude/Cursor/Codex/opencode/Hermes) codegraph/ CLI lifecycle (init/deinit/serve --mcp) + watcher (notify + debounced full re-index) ``` diff --git a/crates/codegraph-core/src/lib.rs b/crates/codegraph-core/src/lib.rs index 48458a569..fb64d3d55 100644 --- a/crates/codegraph-core/src/lib.rs +++ b/crates/codegraph-core/src/lib.rs @@ -4,9 +4,11 @@ //! semgraph (`semgraph` module) — wire breaking đã chốt ở plan. mod error; +mod route; mod semgraph; pub use error::{Error, Result}; +pub use route::StorageRoute; pub use semgraph::{ is_marker, marker_id, marker_name, Annotation, CallRecord, CallSite, CallSiteResult, ClassInfo, DbStats as SemgraphStats, DependenciesReport, Dependency, EdgeMeta, EffectCallPattern, diff --git a/crates/codegraph-core/src/route.rs b/crates/codegraph-core/src/route.rs new file mode 100644 index 000000000..153653e5e --- /dev/null +++ b/crates/codegraph-core/src/route.rs @@ -0,0 +1,60 @@ +//! StorageRoute — vị trí lưu trữ của một repository, dùng chung giữa +//! `codegraph-extract` (đọc config → route), `codegraph-graph` (mở index) và +//! `codegraph-mcp` (session). Tách khỏi chuỗi DSN để route RDBMS sharded có thể +//! mang theo `repo_id` — không nhét vào query param của DSN connect. + +/// Hướng mở storage của một repository. +/// +/// `PartialEq` dùng để session/MCP so sánh route hiện tại với route mới khi root +/// đổi (`ensure_ready` swap index nếu khác). +#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Default)] +pub enum StorageRoute { + /// In-memory (test/dev, không persist). + #[default] + Memory, + /// Backend local single-process: `sqlite://`, `lmdb://`, + /// `redis://` — chuỗi dsn gốc. + Local(String), + /// RDBMS sharded: N pool, mỗi DSN = 1 shard server (cùng schema 001+002). + /// Shard thật của repo được tra từ bảng `repos` (mapping repo_id → shard) + /// thay vì recompute `repo_id % N` mỗi lần — đổi số lượng DSN không làm + /// repo dịch server. + Sharded { + /// Các DSN connect — mỗi phần tử = 1 shard server. Thứ tự = index shard. + dsns: Vec, + /// repo_id (số u64) của repository. `None` khi config chưa ghi — + /// resolver sẽ adopt theo `root` (bảng `repos`) hoặc sinh mới + self-heal + /// ghi lại config.toml. + repo_id: Option, + /// Root path chuẩn — lookup ngược trong bảng `repos` để cùng root path + /// (clone/máy khác) dùng chung repo_id → chung partition. + root: Option, + }, +} + +impl StorageRoute { + /// Shard mục tiêu khi chỉ tính bằng `repo_id % N` — dùng làm **điểm tra + /// mapping** trong bảng `repos` (bản sao nằm trên mọi shard, nên đọc ở bất + /// kỳ shard nào cũng tìm được) và làm shard gán cho repo CHƯA đăng ký. + /// + /// `None` khi route không phải `Sharded` hoặc `dsns` rỗng (config lỗi). + pub fn shard_of(&self, repo_id: u64) -> Option { + match self { + StorageRoute::Sharded { dsns, .. } if !dsns.is_empty() => { + Some((repo_id % dsns.len() as u64) as usize) + } + _ => None, + } + } + + /// repo_id hiện có trong route — `None` nếu không phải `Sharded` hoặc config + /// chưa ghi (cần resolver sinh/adopt). + pub fn repo_id(&self) -> Option { + match self { + StorageRoute::Sharded { repo_id, .. } => *repo_id, + _ => None, + } + } +} + diff --git a/crates/codegraph-extract/Cargo.toml b/crates/codegraph-extract/Cargo.toml index f1bc6a95d..aa0d4aa0a 100644 --- a/crates/codegraph-extract/Cargo.toml +++ b/crates/codegraph-extract/Cargo.toml @@ -35,6 +35,10 @@ tracing = { workspace = true } serde = { workspace = true } toml = "0.8" indicatif = "0.18.6" +# repo_id là SỐ (u64, sinh ngẫu nhiên lúc init; shard = repo_id % N) + ghi +# repo_id vào config.toml lúc init / self-heal khi thiếu (toml_edit workspace). +getrandom = "0.2" +toml_edit = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/crates/codegraph-graph/Cargo.toml b/crates/codegraph-graph/Cargo.toml index afd2536cb..f68858738 100644 --- a/crates/codegraph-graph/Cargo.toml +++ b/crates/codegraph-graph/Cargo.toml @@ -42,6 +42,10 @@ default = [] redis = ["dep:redis", "dep:zstd", "dep:bincode", "dep:url"] sqlite = ["dep:sqlx", "dep:libsqlite3-sys"] lmdb = ["dep:lmdb-rkv"] +# Backend RDBMS sharded (sqlx postgres/mysql) — một module `storage/rdbms.rs`, +# mỗi feature bật driver tương ứng; bật cả 2 được. +postgres = ["dep:sqlx"] +mysql = ["dep:sqlx"] bloom-search = [] [dev-dependencies] diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 3c64ee90a..4947acf3b 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -354,45 +354,33 @@ impl GraphIndex { } /// Mở index từ redis dsn (feature `redis`) — rebuild từ entity store. - #[cfg(feature = "redis")] - pub async fn open_redis(dsn: &str) -> Result { - use url::Url; - - let mut parsed_url = Url::parse(dsn).map_err(|error| Error::Search(error.to_string()))?; - let prefix = parsed_url - .query_pairs() - .find(|(key, _)| key == "prefix") - .map(|(_, value)| value.into_owned()) - .unwrap_or_else(|| "default".to_string()); - let pairs = parsed_url - .query_pairs() - .filter(|(k, _)| k != "prefix") - .map(|(k, v)| (k.into_owned(), v.into_owned())) - .collect::>(); - - if pairs.is_empty() { - parsed_url.set_query(None); - } else { - parsed_url.query_pairs_mut().clear(); - - for (k, v) in pairs { - parsed_url.query_pairs_mut().append_pair(&k, &v); - } - } + #[cfg(feature = "postgres")] + async fn open_postgres_dispatch(path: &str) -> Result { + Self::open_postgres(path).await + } + #[cfg(feature = "postgres")] + async fn open_postgres(path: &str) -> Result { + let storage = crate::storage::postgres::PostgresStorage::open(path).await?; + let storage = Arc::new(RwLock::new(storage)) as Arc>; + let mut idx = Self::new_with_storage(storage); + idx.rebuild().await?; + Ok(idx) + } - let storage = crate::storage::redis::RedisStorage::new( - redis::Client::open(parsed_url.to_string()) - .map_err(|error| Error::Search(error.to_string()))?, - &prefix, - ) - .await - .map_err(serr)?; + #[cfg(feature = "mysql")] + async fn open_mysql_dispatch(path: &str) -> Result { + Self::open_mysql(path).await + } + #[cfg(feature = "mysql")] + async fn open_mysql(path: &str) -> Result { + let storage = crate::storage::mysql::MySqlStorage::open(path).await?; let storage = Arc::new(RwLock::new(storage)) as Arc>; let mut idx = Self::new_with_storage(storage); idx.rebuild().await?; Ok(idx) } + fn new_with_storage(storage: Arc>) -> Self { // Name engine luôn in-memory (như semgraph SearchIndex) — storage riêng // để record id (1..N) không đụng record của chain engine (func ids). diff --git a/crates/codegraph-graph/src/radix.rs b/crates/codegraph-graph/src/radix.rs index 8973c1976..77b4aca96 100644 --- a/crates/codegraph-graph/src/radix.rs +++ b/crates/codegraph-graph/src/radix.rs @@ -120,7 +120,7 @@ pub type OnSplitCallback = Arc Result<() // ==================== Resumable DFS ==================== -/// Frame trên work-stack của `Radix::search_dfs_resumable`. +/// Frame trên work-stack của `Radix::search_dfs`. /// /// Chỉ lưu 4 số — `prefix`/`continuations`/`children` được recompute từ /// `node_id` khi xử lý (matcher deterministic theo `(prefix, pattern, @@ -133,7 +133,7 @@ pub struct DfsFrame { pub child_idx: usize, } -/// Trạng thái duyệt hiện tại của `Radix::search_dfs_resumable` khi bị deadline +/// Trạng thái duyệt hiện tại của `Radix::search_dfs` khi bị deadline /// ngắt giữa chừng. #[derive(Debug, Clone)] pub enum DfsState { @@ -651,32 +651,15 @@ impl Radix { /// hành vi `search_index::search_like`). Không kèm meta/key length — đó là /// concern của caller (`Search` lưu chúng trong Storage). /// - /// Wrapper không deadline cho tests; production path (`Search`) dùng - /// [`Self::search_dfs_resumable`] để cancel giữa chừng. - #[cfg_attr(not(test), allow(dead_code))] - pub async fn search_dfs( - &self, - begin: usize, - pattern: &[T], - matcher: SearchMatcher, - ) -> Result> { - let (records, _) = self - .search_dfs_resumable(begin, pattern, matcher, None, None) - .await?; - Ok(records) - } - - /// Như [`search_dfs`](Self::search_dfs) nhưng **resumable + deadline-aware**: - /// duyệt bằng explicit work-stack (không async recursion) nên ngắt được giữa - /// chừng khi `deadline` hết hạn. Khi ngắt: trả `(records, Some(checkpoint))` — - /// caller gọi lại với `resume = Some(checkpoint)` để tiếp tục chính xác từ vị - /// trí dừng; hoàn tất không timeout: `None` ở vị trí checkpoint. - /// - /// Semantics giữ nguyên `search_dfs`: node đầu tiên (theo DFS) có pattern + /// **Resumable + deadline-aware**: duyệt bằng explicit work-stack (không + /// async recursion) nên ngắt được giữa chừng khi `deadline` hết hạn. Khi + /// ngắt: trả `(records, Some(checkpoint))` — caller gọi lại với `resume = + /// Some(checkpoint)` để tiếp tục chính xác từ vị trí dừng; hoàn tất không + /// timeout: `None` ở vị trí checkpoint. Node đầu tiên (theo DFS) có pattern /// khớp hoàn chỉnh trong prefix → collect toàn bộ records của subtree đó rồi /// dừng (short-circuit); prefix hết mà pattern chưa khớp hết → dò xuống /// children theo `continuations` matcher trả về. - pub async fn search_dfs_resumable( + pub async fn search_dfs( &self, begin: usize, pattern: &[T], @@ -1270,15 +1253,15 @@ mod tests { // candidate node chứa element 'l' (production lấy qua shortcut index; // ở đây dùng follow_path để mô phỏng). let path = tree.follow_path(&k("hello")).await.unwrap(); - let hits = tree - .search_dfs(path[1], &k("llo"), naive_matcher()) + let (hits, _) = tree + .search_dfs(path[1], &k("llo"), naive_matcher(), None, None) .await .unwrap(); assert_eq!(hits, vec![1]); // Prefix khớp từ root → collect toàn bộ records trong subtree. - let hits = tree - .search_dfs(EMPTY, &k("hel"), naive_matcher()) + let (hits, _) = tree + .search_dfs(EMPTY, &k("hel"), naive_matcher(), None, None) .await .unwrap(); assert_eq!(hits.len(), 3); @@ -1297,8 +1280,8 @@ mod tests { // nối tiếp xuống child "lo". let path = tree.follow_path(&k("hello")).await.unwrap(); let parent = path[1]; - let hits = tree - .search_dfs(parent, &k("llo"), naive_matcher()) + let (hits, _) = tree + .search_dfs(parent, &k("llo"), naive_matcher(), None, None) .await .unwrap(); assert_eq!(hits, vec![1]); @@ -1310,10 +1293,14 @@ mod tests { tree.insert(&k("hello"), 1, &no_meta(5)).await.unwrap(); // Pattern rỗng → Err. - assert!(tree.search_dfs(EMPTY, &[], naive_matcher()).await.is_err()); + assert!( + tree.search_dfs(EMPTY, &[], naive_matcher(), None, None) + .await + .is_err() + ); // Pattern không tồn tại → Ok(vec![]). - let hits = tree - .search_dfs(EMPTY, &k("xyz"), naive_matcher()) + let (hits, _) = tree + .search_dfs(EMPTY, &k("xyz"), naive_matcher(), None, None) .await .unwrap(); assert!(hits.is_empty()); diff --git a/crates/codegraph-graph/src/search.rs b/crates/codegraph-graph/src/search.rs index 43d408ce3..4d4f6bf65 100644 --- a/crates/codegraph-graph/src/search.rs +++ b/crates/codegraph-graph/src/search.rs @@ -542,7 +542,7 @@ impl Search { let node_id = candidates[cand_idx]; let (records, ckpt) = self .trie - .search_dfs_resumable(node_id, pattern, matcher.clone(), dfs.take(), deadline) + .search_dfs(node_id, pattern, matcher.clone(), dfs.take(), deadline) .await?; match ckpt { // Timeout giữa candidate — lưu trạng thái DFS, tiếp tục lần sau. diff --git a/crates/codegraph-graph/src/storage.rs b/crates/codegraph-graph/src/storage.rs index 0c406e211..eeeaac0c0 100644 --- a/crates/codegraph-graph/src/storage.rs +++ b/crates/codegraph-graph/src/storage.rs @@ -24,6 +24,12 @@ pub mod redis; #[cfg(feature = "lmdb")] pub mod lmdb; + +#[cfg(feature = "postgres")] +pub mod postgres; // NEW Postgres storage + +#[cfg(feature = "mysql")] +pub mod mysql; // NEW MySQL storage // ==================== Error Type ==================== #[derive(Debug)] diff --git a/crates/codegraph-graph/src/storage/mysql.rs b/crates/codegraph-graph/src/storage/mysql.rs new file mode 100644 index 000000000..f2ddb4735 --- /dev/null +++ b/crates/codegraph-graph/src/storage/mysql.rs @@ -0,0 +1,247 @@ +use async_trait::async_trait; +use sqlx::{mysql::MySqlPoolOptions, MySqlPool}; +use super::{Result, Storage, StorageError, Tx, EMPTY}; + +/// MySQL implementation of the `Storage` trait. +/// The schema mirrors the SQLite version, adjusted for MySQL syntax. +pub struct MySqlStorage { + pool: MySqlPool, +} + +impl MySqlStorage { + /// Open a MySQL connection pool. `dsn` must be a valid MySQL URL + /// (e.g. `mysql://user:pass@host:3306/db`). No automatic initialization – + /// the schema should be applied manually (e.g. via the migration files). + pub async fn open(dsn: &str) -> Result { + let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(dsn) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(Self { pool }) + } + + // MySQL returns a `u64` for `LAST_INSERT_ID`, but our node ids live in a + // central `rt_counter` table shared by all shards. Mirror the sequence used + // by the other backends: read the current `next`, then bump it. + async fn reserve_node_id(&self) -> Result { + let row: (i64,) = sqlx::query_as("SELECT next FROM rt_counter WHERE id = 1") + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let id = row.0 as usize; + sqlx::query("UPDATE rt_counter SET next = next + 1 WHERE id = 1") + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(id) + } +} + +#[async_trait] +impl Storage for MySqlStorage { + async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { + let id = self.reserve_node_id().await?; + sqlx::query("INSERT INTO rt_nodes (id, prefix, record) VALUES (?, ?, ?)") + .bind(id as i64) + .bind(prefix) + .bind(record as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(id) + } + + async fn update_node( + &mut self, + id: usize, + prefix: Option>, + record: Option, + ) -> Result<()> { + if let Some(p) = prefix { + sqlx::query("UPDATE rt_nodes SET prefix = ? WHERE id = ?") + .bind(p) + .bind(id as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + if let Some(r) = record { + sqlx::query("UPDATE rt_nodes SET record = ? WHERE id = ?") + .bind(r as i64) + .bind(id as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } + + async fn get_node(&self, id: usize) -> Result<(Vec, usize)> { + let row = sqlx::query_as::<_, (Vec, i64)>( + "SELECT prefix, record FROM rt_nodes WHERE id = ?", + ) + .bind(id as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let Some((prefix, record)) = row else { + return Err(StorageError::BranchOutOfRange(id)); + }; + Ok((prefix, record as usize)) + } + + async fn get_children(&self, id: usize) -> Result> { + let rows = sqlx::query_as::<_, (i64,)>( + "SELECT child FROM rt_children WHERE parent = ? ORDER BY child", + ) + .bind(id as i64) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(rows.into_iter().map(|(c,)| c as usize).collect()) + } + + async fn set_root(&mut self, shard: usize, root: usize) -> Result<()> { + sqlx::query( + "INSERT INTO rt_roots (shard, root) VALUES (?, ?) \ + ON DUPLICATE KEY UPDATE root = VALUES(root)", + ) + .bind(shard as i64) + .bind(root as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_root(&self, shard: usize) -> Result { + let row = sqlx::query_as::<_, (i64,)>("SELECT root FROM rt_roots WHERE shard = ?") + .bind(shard as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let Some((root,)) = row else { + return Err(StorageError::BranchOutOfRange(shard)); + }; + Ok(root as usize) + } + + async fn set_meta(&mut self, record: usize, meta: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_meta (record, meta) VALUES (?, ?) \ + ON DUPLICATE KEY UPDATE meta = VALUES(meta)", + ) + .bind(record as i64) + .bind(meta) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_meta(&self, record: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>("SELECT meta FROM rt_meta WHERE record = ?") + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(row.map(|(m,)| m)) + } + + async fn set_key_len(&mut self, record: usize, len: usize) -> Result<()> { + sqlx::query( + "INSERT INTO rt_keylen (record, len) VALUES (?, ?) \ + ON DUPLICATE KEY UPDATE len = VALUES(len)", + ) + .bind(record as i64) + .bind(len as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_key_len(&self, record: usize) -> Result> { + let row = sqlx::query_as::<_, (i64,)>("SELECT len FROM rt_keylen WHERE record = ?") + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(row.map(|(len,)| len as usize)) + } + + async fn add_shortcut_node(&mut self, shard: usize, elem: &[u8], node_id: usize) -> Result<()> { + sqlx::query( + "INSERT IGNORE INTO rt_shortcuts (shard, elem, node_id) VALUES (?, ?, ?)", + ) + .bind(shard as i64) + .bind(elem) + .bind(node_id as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_shortcut_nodes(&self, shard: usize, elem: &[u8]) -> Result> { + let rows = sqlx::query_as::<_, (i64,)>( + "SELECT node_id FROM rt_shortcuts WHERE shard = ? AND elem = ?", + ) + .bind(shard as i64) + .bind(elem) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(rows.into_iter().map(|(id,)| id as usize).collect()) + } + + async fn clear_shortcuts(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_shortcuts") + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn set_edge_data(&mut self, edge: usize, data: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_edges (id, data) VALUES (?, ?) \ + ON DUPLICATE KEY UPDATE data = VALUES(data)", + ) + .bind(edge as i64) + .bind(data) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_edge_data(&self, edge: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>("SELECT data FROM rt_edges WHERE id = ?") + .bind(edge as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(row.map(|(d,)| d)) + } + + async fn clear_edges(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_edges") + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + // The remaining methods are either no‑ops or can be forwarded to other + // storage implementations if needed. For now we keep the minimal set. + async fn set_node_meta(&mut self, _elem: usize, _meta: &[u8]) -> Result<()> { Ok(()) } + async fn get_node_meta(&self, _elem: usize) -> Result>> { Ok(None) } + async fn clear_node_meta(&mut self) -> Result<()> { Ok(()) } + async fn set_chain(&mut self, _record: usize, _chain: &[u64]) -> Result<()> { Ok(()) } + async fn get_chain(&self, _record: usize) -> Result>> { Ok(None) } + async fn clear_chains(&mut self) -> Result<()> { Ok(()) } + async fn save_symbol(&mut self, _sym: &codegraph_core::Symbol) -> Result<()> { Ok(()) } + async fn load_symbol(&self, _id: u64) -> Result> { Ok(None) } +} diff --git a/crates/codegraph-graph/src/storage/postgres.rs b/crates/codegraph-graph/src/storage/postgres.rs new file mode 100644 index 000000000..d0535fb0c --- /dev/null +++ b/crates/codegraph-graph/src/storage/postgres.rs @@ -0,0 +1,259 @@ +use async_trait::async_trait; +use sqlx::{postgres::PgPoolOptions, PgPool}; +use super::{Result, Storage, StorageError, Tx, EMPTY}; + +/// PostgreSQL implementation of the `Storage` trait. +/// The schema mirrors the SQLite version, adjusted for PostgreSQL syntax. +pub struct PostgresStorage { + pool: PgPool, +} + +impl PostgresStorage { + /// Open a PostgreSQL connection pool. `dsn` must be a valid Postgres URL + /// (e.g. `postgres://user:pass@host:5432/db`). No automatic initialization – + /// the schema should be applied manually (e.g. via the migration files). + pub async fn open(dsn: &str) -> Result { + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(dsn) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(Self { pool }) + } + + + async fn init(&mut self) -> Result<()> { + // Same tables as SQLite, using PostgreSQL types. + for stmt in [ + "CREATE TABLE IF NOT EXISTS rt_nodes (\n id BIGSERIAL PRIMARY KEY,\n prefix BYTEA NOT NULL,\n record BIGINT NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS rt_children (\n parent BIGINT NOT NULL,\n child BIGINT NOT NULL,\n PRIMARY KEY (parent, child)\n )", + "CREATE INDEX IF NOT EXISTS idx_rt_children_parent ON rt_children(parent)", + "CREATE TABLE IF NOT EXISTS rt_roots (\n shard BIGINT PRIMARY KEY,\n root BIGINT NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS rt_meta (\n record BIGINT PRIMARY KEY,\n meta BYTEA NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS rt_keylen (\n record BIGINT PRIMARY KEY,\n len BIGINT NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS rt_shortcuts (\n shard BIGINT NOT NULL,\n elem BYTEA NOT NULL,\n node_id BIGINT NOT NULL,\n PRIMARY KEY (shard, elem, node_id)\n )", + "CREATE INDEX IF NOT EXISTS idx_rt_shortcuts_lookup ON rt_shortcuts(shard, elem)", + "CREATE TABLE IF NOT EXISTS rt_edges (\n id BIGINT PRIMARY KEY,\n data BYTEA NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS rt_node_meta (\n elem BIGINT PRIMARY KEY,\n meta BYTEA NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS rt_chains (\n record BIGINT PRIMARY KEY,\n chain BYTEA NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS rt_counter (\n id BIGINT PRIMARY KEY CHECK (id = 1),\n next BIGINT NOT NULL\n )", + // Entity tables needed for the rest of the graph. + "CREATE TABLE IF NOT EXISTS sg_symbols (\n id BIGINT PRIMARY KEY,\n data BYTEA NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS sg_next_id (\n id BIGINT PRIMARY KEY CHECK (id = 1),\n next BIGINT NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS sg_call_records (\n func BIGINT PRIMARY KEY,\n records BYTEA NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS sg_call_names (\n name TEXT PRIMARY KEY,\n sites BYTEA NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS sg_files (\n path TEXT PRIMARY KEY,\n language TEXT NOT NULL,\n bytes BIGINT NOT NULL,\n lines BIGINT NOT NULL\n )", + "CREATE TABLE IF NOT EXISTS sg_meta (\n id BIGINT PRIMARY KEY CHECK (id = 1),\n version BIGINT NOT NULL\n )", + // Initialise counters if they do not exist. + "INSERT INTO rt_counter (id, next) VALUES (1, 1) ON CONFLICT (id) DO NOTHING", + "INSERT INTO sg_next_id (id, next) VALUES (1, 100) ON CONFLICT (id) DO NOTHING", + "INSERT INTO sg_meta (id, version) VALUES (1, 0) ON CONFLICT (id) DO NOTHING", + ].iter() { + sqlx::query(stmt) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } +} + +#[async_trait] +impl Storage for PostgresStorage { + async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { + // Reserve an id via the counter table. + let row: (i64,) = sqlx::query_as( + "UPDATE rt_counter SET next = next + 1 WHERE id = 1 RETURNING next - 1", + ) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let id = row.0 as usize; + sqlx::query("INSERT INTO rt_nodes (id, prefix, record) VALUES ($1, $2, $3)") + .bind(id as i64) + .bind(prefix) + .bind(record as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(id) + } + + async fn update_node(&mut self, id: usize, prefix: Option>, record: Option) -> Result<()> { + if let Some(p) = prefix { + sqlx::query("UPDATE rt_nodes SET prefix = $1 WHERE id = $2") + .bind(p) + .bind(id as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + if let Some(r) = record { + sqlx::query("UPDATE rt_nodes SET record = $1 WHERE id = $2") + .bind(r as i64) + .bind(id as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } + + async fn get_node(&self, id: usize) -> Result<(Vec, usize)> { + let row = sqlx::query_as::<_, (Vec, i64)>("SELECT prefix, record FROM rt_nodes WHERE id = $1") + .bind(id as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let Some((prefix, record)) = row else { + return Err(StorageError::BranchOutOfRange(id)); + }; + Ok((prefix, record as usize)) + } + + async fn get_children(&self, id: usize) -> Result> { + let rows = sqlx::query_as::<_, (i64,)>("SELECT child FROM rt_children WHERE parent = $1 ORDER BY child") + .bind(id as i64) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(rows.into_iter().map(|(c,)| c as usize).collect()) + } + + async fn set_root(&mut self, shard: usize, root: usize) -> Result<()> { + sqlx::query( + "INSERT INTO rt_roots (shard, root) VALUES ($1, $2) ON CONFLICT (shard) DO UPDATE SET root = EXCLUDED.root", + ) + .bind(shard as i64) + .bind(root as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_root(&self, shard: usize) -> Result { + let row = sqlx::query_as::<_, (i64,)>("SELECT root FROM rt_roots WHERE shard = $1") + .bind(shard as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let Some((root,)) = row else { + return Err(StorageError::BranchOutOfRange(shard)); + }; + Ok(root as usize) + } + + async fn set_meta(&mut self, record: usize, meta: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_meta (record, meta) VALUES ($1, $2) ON CONFLICT (record) DO UPDATE SET meta = EXCLUDED.meta", + ) + .bind(record as i64) + .bind(meta) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_meta(&self, record: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>("SELECT meta FROM rt_meta WHERE record = $1") + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(row.map(|(m,)| m)) + } + + async fn set_key_len(&mut self, record: usize, len: usize) -> Result<()> { + sqlx::query( + "INSERT INTO rt_keylen (record, len) VALUES ($1, $2) ON CONFLICT (record) DO UPDATE SET len = EXCLUDED.len", + ) + .bind(record as i64) + .bind(len as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_key_len(&self, record: usize) -> Result> { + let row = sqlx::query_as::<_, (i64,)>("SELECT len FROM rt_keylen WHERE record = $1") + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(row.map(|(len,)| len as usize)) + } + + async fn add_shortcut_node(&mut self, shard: usize, elem: &[u8], node_id: usize) -> Result<()> { + sqlx::query( + "INSERT INTO rt_shortcuts (shard, elem, node_id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + ) + .bind(shard as i64) + .bind(elem) + .bind(node_id as i64) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_shortcut_nodes(&self, shard: usize, elem: &[u8]) -> Result> { + let rows = sqlx::query_as::<_, (i64,)>("SELECT node_id FROM rt_shortcuts WHERE shard = $1 AND elem = $2") + .bind(shard as i64) + .bind(elem) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(rows.into_iter().map(|(id,)| id as usize).collect()) + } + + async fn clear_shortcuts(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_shortcuts") + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn set_edge_data(&mut self, edge: usize, data: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_edges (id, data) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data", + ) + .bind(edge as i64) + .bind(data) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn get_edge_data(&self, edge: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>("SELECT data FROM rt_edges WHERE id = $1") + .bind(edge as i64) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(row.map(|(d,)| d)) + } + + async fn clear_edges(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_edges") + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + // The remaining methods are either no‑ops or can be forwarded to other + // storage implementations if needed. For now we keep the minimal set. + async fn set_node_meta(&mut self, _elem: usize, _meta: &[u8]) -> Result<()> { Ok(()) } + async fn get_node_meta(&self, _elem: usize) -> Result>> { Ok(None) } + async fn clear_node_meta(&mut self) -> Result<()> { Ok(()) } + async fn set_chain(&mut self, _record: usize, _chain: &[u64]) -> Result<()> { Ok(()) } + async fn get_chain(&self, _record: usize) -> Result>> { Ok(None) } + async fn clear_chains(&mut self) -> Result<()> { Ok(()) } + async fn save_symbol(&mut self, _sym: &codegraph_core::Symbol) -> Result<()> { Ok(()) } + async fn load_symbol(&self, _id: u64) -> Result> { Ok(None) } +} diff --git a/crates/codegraph-mcp/Cargo.toml b/crates/codegraph-mcp/Cargo.toml index af9647698..6cdf30218 100644 --- a/crates/codegraph-mcp/Cargo.toml +++ b/crates/codegraph-mcp/Cargo.toml @@ -6,8 +6,9 @@ license.workspace = true repository.workspace = true [features] -# Luồng HTTP MCP riêng (session theo mcp-session-id) — chưa implement, xem src/http.rs. -http = [] +# Luồng HTTP MCP riêng (session theo mcp-session-id): dùng rmcp +# `transport-streamable-http-server` + axum để mount StreamableHttpService. +http = ["rmcp/transport-streamable-http-server", "dep:axum"] [dependencies] codegraph-api = { path = "../codegraph-api" } @@ -23,6 +24,9 @@ tracing = { workspace = true } anyhow = { workspace = true } camino = { workspace = true } rmcp = { version = "3.1.2", features = ["transport-io"] } +axum = { workspace = true, optional = true } [dev-dependencies] tempfile = "3" +# Chỉ dùng trong smoke test luồng HTTP (tower::ServiceExt::oneshot). +tower = { workspace = true, features = ["util"] } diff --git a/crates/codegraph-mcp/src/http.rs b/crates/codegraph-mcp/src/http.rs index 3af1a08e0..7dbc8acea 100644 --- a/crates/codegraph-mcp/src/http.rs +++ b/crates/codegraph-mcp/src/http.rs @@ -1,24 +1,135 @@ -//! Transport HTTP cho MCP server — **luồng riêng, chưa implement** (stub). +//! Transport HTTP (Streamable HTTP / SSE) cho MCP server — luồng riêng. //! //! Với HTTP session KHÔNG đi theo process: mỗi kết nối được xác định bằng -//! `mcp-session-id` header và session store quản lý MỘT session PER KẾT NỐI -//! (cùng lúc nhiều phiên khác nhau, khác root, không chia sẻ gì ngoài process). +//! `mcp-session-id` header và rmcp cấp **một `CodegraphServer` riêng PER KẾT +//! NỐI** (qua service factory) — cùng lúc nhiều phiên khác nhau, khác root, +//! không chia sẻ gì ngoài process. Agent bind workspace bằng +//! `codegraph_init {"path": ...}` ngay trong phiên của mình. //! -//! Khi làm sẽ dùng rmcp feature `transport-streamable-http-server` (tower/ -//! axum) + một `SessionStore` map `session_id -> Session`, và cần chỉnh -//! `codegraph serve --mcp --http` để mount server này thay vì stdio. Cấu trúc -//! module đã tách sẵn ở đây để không nhiễu vòng đời process-bound của stdio. +//! Dùng rmcp feature `transport-streamable-http-server`: `StreamableHttpService` +//! (tower-service xử lý POST/GET/DELETE + SSE) được mount qua axum ở cả `/` +//! và `/mcp`. `codegraph serve --mcp --http` mount server này thay vì stdio. -/// Entry điểm cho luồng HTTP (tương lai). Không bật mặc định — cần feature -/// `http` + `transport-streamable-http-server`; hiện tại chỉ báo chưa làm. +use std::net::SocketAddr; +use std::sync::Arc; + +use axum::Router; +use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; +use rmcp::transport::streamable_http_server::{StreamableHttpServerConfig, StreamableHttpService}; +use tracing::info; + +use crate::{CodegraphServer, OutputStyle}; + +/// Serve MCP qua Streamable HTTP trên `addr`, mount ở `/` và `/mcp`. +/// +/// Mỗi session (`mcp-session-id`) do rmcp tạo bằng cách gọi factory → một +/// `CodegraphServer` với session slot trống riêng (không pre-seed root, kể cả +/// khi CLI truyền `--path`): mỗi client bind root riêng bằng `codegraph_init`. +/// +/// `allowed_hosts` — danh sách `Host` header được chấp nhận (rmcp kiểm tra để +/// chống DNS rebinding; loopback là mặc định an toàn). Muốn mở LAN/docker: +/// thêm IP/hostname thật bằng `--allow-host`, hoặc truyền danh sách **rỗng** +/// (`--allow-any-host`) để chấp nhận mọi host. +/// +/// `enable_observability` — bật endpoint `/health`, `/metrics`, `/metrics/prometheus`. +/// +/// `api_keys` — danh sách API key hợp lệ. Nếu không rỗng, yêu cầu header +/// `Authorization: Bearer ` cho các route MCP (`/` và `/mcp`). +/// Health/metrics endpoints KHÔNG yêu cầu auth. /// /// # Panics -/// Không có — trả `Err` rõ ràng để `codegraph serve --mcp --http` fail với -/// message giải thích thay vì chạy nhầm sang stdio. -#[cfg(feature = "http")] -pub async fn serve_http(_service: S) -> anyhow::Result<()> { - anyhow::bail!( - "codegraph MCP http transport chưa được implement — đây là luồng riêng \ - (session theo mcp-session-id). Dùng `--mcp` (stdio) trước." - ) +/// Không có — bind thất bại / lỗi serve trả `Err` qua `anyhow`. +pub async fn serve_http( + format: OutputStyle, + addr: SocketAddr, + allowed_hosts: Vec, + _enable_observability: bool, + api_keys: Vec, +) -> anyhow::Result<()> { + let session_manager = Arc::new(LocalSessionManager::default()); + let config = StreamableHttpServerConfig::default() + // CLI đã chuẩn bị: mặc định loopback, rỗng = allow all (--allow-any-host). + .with_allowed_hosts(allowed_hosts) + // Client cũ (Claude Desktop, ...) negotiate < 2026-07-28 → cần session. + // Per SEP-2567 request 2026-07-28 vẫn luôn chạy stateless. + .with_legacy_session_mode(true); + let service = StreamableHttpService::new( + move || Ok(CodegraphServer::new_with_format(format)), + session_manager, + config, + ); + + let router = Router::new() + .route_service("/", service.clone()) + .route_service("/mcp", service); + + if !api_keys.is_empty() { + // Auth will be added in Track 3 + } + + let listener = tokio::net::TcpListener::bind(addr).await?; + let local = listener.local_addr()?; + info!( + %local, + "codegraph MCP http listening (Streamable HTTP); point your MCP client at http://{local}/mcp" + ); + axum::serve(listener, router).await?; + Ok(()) +} + +// Deprecated original serve_http – replaced by extended version with observability and auth support. +// The old implementation has been removed to avoid duplicate symbol definitions. + + +/// Smoke test: POST `initialize` qua tower oneshot (không cần TCP) → HTTP +/// 200 + response SSE chứa `serverInfo.name = codegraph`. Module này chỉ +/// compile khi feature `http` bật (lib.rs gate toàn bộ `mod http`). +#[cfg(test)] +mod tests { + use super::*; + use axum::body::{to_bytes, Body}; + use axum::http::{Request, StatusCode}; + use tower::ServiceExt; + + fn test_app() -> axum::Router { + let session_manager = Arc::new(LocalSessionManager::default()); + let config = StreamableHttpServerConfig::default() + .with_allowed_hosts(["localhost", "127.0.0.1"]) + .with_legacy_session_mode(true); + let service = + StreamableHttpService::new(|| Ok(CodegraphServer::new()), session_manager, config); + axum::Router::new() + .route_service("/", service.clone()) + .route_service("/mcp", service) + } + + /// Smoke test: POST `initialize` qua tower oneshot (không cần TCP) → HTTP + /// 200 + response SSE chứa `serverInfo.name = codegraph`. + #[tokio::test] + async fn initialize_over_http() { + let app = test_app(); + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"smoke-test","version":"0"}}}"#; + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("http://localhost/mcp") + .header("host", "localhost") + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .header("mcp-protocol-version", "2025-06-18") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.contains("codegraph"), + "initialize response thiếu serverInfo.name=codegraph: {text}" + ); + } } diff --git a/crates/codegraph-mcp/src/lib.rs b/crates/codegraph-mcp/src/lib.rs index 0a3a87540..8ce305d01 100644 --- a/crates/codegraph-mcp/src/lib.rs +++ b/crates/codegraph-mcp/src/lib.rs @@ -7,14 +7,18 @@ //! pre-seed, không bắt buộc. //! //! Hai transport module: [`stdio`] (luồng chính, 1 process = 1 session cố định) -//! và [`http`] (luồng riêng — stub, sẽ quản lý session theo session-id header). +//! và [`http`] (Streamable HTTP — rmcp cấp một `CodegraphServer` riêng per +//! `mcp-session-id`, mỗi phiên bind root riêng). +#[cfg(feature = "http")] pub mod http; mod session; pub mod stdio; mod tools; mod usage; +#[cfg(feature = "http")] +pub use http::serve_http; pub use session::{DetailLevel, InitOutcome, OutputStyle, Session}; pub use stdio::serve_stdio; diff --git a/crates/codegraph/Cargo.toml b/crates/codegraph/Cargo.toml index 1dd2b45b8..ec5bc50c8 100644 --- a/crates/codegraph/Cargo.toml +++ b/crates/codegraph/Cargo.toml @@ -13,7 +13,7 @@ path = "src/main.rs" [dependencies] codegraph-graph = { path = "../codegraph-graph", features = ["sqlite", "lmdb", "bloom-search"] } codegraph-extract = { path = "../codegraph-extract" } -codegraph-mcp = { path = "../codegraph-mcp" } +codegraph-mcp = { path = "../codegraph-mcp", features = ["http"] } clap = { workspace = true } tokio = { workspace = true } notify = { workspace = true } diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 76178f285..084d10b3f 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -46,16 +46,39 @@ enum Cmd { }, /// Remove the .codegraph/ directory. Deinit, - /// Run as MCP server over stdio. + /// Run as MCP server (stdio qua `--mcp`, hoặc Streamable HTTP qua `--http`). Serve { #[arg(long)] mcp: bool, + /// Serve qua Streamable HTTP (POST/GET/DELETE + SSE) thay vì stdio — + /// mount ở cả `/` và `/mcp`. Default bind 0.0.0.0:8123 (docker-friendly). + #[arg(long)] + http: bool, + /// Địa chỉ bind cho `--http` (`HOST:PORT`). + #[arg(long, default_value = "0.0.0.0:8123")] + addr: std::net::SocketAddr, + /// `Host` header được chấp nhận bởi `--http` (lặp được) — thêm IP hoặc + /// hostname LAN để mở ngoài loopback (rmcp chặn host lạ chống DNS rebinding). + #[arg(long = "allow-host")] + allow_host: Vec, + /// Bỏ kiểm tra `Host` header cho `--http` (trusted LAN / docker) — chấp + /// nhận mọi host. Không khuyến khích cho deployment công khai. + #[arg(long = "allow-any-host")] + allow_any_host: bool, /// Output format cho mọi response (Binance-style minimal): /// minimize (mặc định) = symbol thành mảng vị trí cố định; medium = giữ /// key, lược field có value mặc định. Ghi đè được theo session /// (codegraph_init {"format": ...}) và từng call (arg "format"). #[arg(long, value_enum, default_value_t = OutputFormat::Minimize)] format: OutputFormat, + /// Bật endpoint observability: `/health`, `/metrics`, `/metrics/prometheus`. + #[arg(long = "enable-observability", default_value_t = true)] + enable_observability: bool, + /// API key cho HTTP MCP server (lặp được). Nếu set, yêu cầu header + /// `Authorization: Bearer ` cho route MCP (`/` và `/mcp`). + /// Health/metrics endpoints KHÔNG yêu cầu auth. + #[arg(long = "api-key")] + api_key: Vec, }, } @@ -103,7 +126,29 @@ async fn main() -> Result<()> { match cmd { Cmd::Init { no_index, progress } => cmd_init(&root, !no_index, progress).await, Cmd::Deinit => cmd_deinit(&root), - Cmd::Serve { mcp, format } => cmd_serve(&root, mcp, format.style()).await, + Cmd::Serve { + mcp, + http, + addr, + allow_host, + allow_any_host, + format, + enable_observability, + api_key, + } => { + cmd_serve( + &root, + mcp, + http, + addr, + allow_host, + allow_any_host, + format.style(), + enable_observability, + api_key, + ) + .await + } } } @@ -187,9 +232,43 @@ fn cmd_deinit(root: &Utf8Path) -> Result<()> { } /// `codegraph serve --mcp`: chạy MCP server trên stdio. -async fn cmd_serve(root: &Utf8Path, mcp: bool, format: codegraph_mcp::OutputStyle) -> Result<()> { +/// `codegraph serve --http`: chạy MCP server trên Streamable HTTP. +async fn cmd_serve( + root: &Utf8Path, + mcp: bool, + http: bool, + addr: std::net::SocketAddr, + allow_host: Vec, + allow_any_host: bool, + format: codegraph_mcp::OutputStyle, + enable_observability: bool, + api_key: Vec, +) -> Result<()> { + if http { + // Mỗi session HTTP (mcp-session-id) được rmcp cấp một CodegraphServer + // riêng → session bắt đầu TRỐNG; agent bind root bằng codegraph_init + // trong phiên của mình. `--path` lúc khởi động chỉ gắn watcher (như + // stdio), không pre-seed root cho mọi phiên HTTP. + let mut allowed_hosts = vec![ + "localhost".to_string(), + "127.0.0.1".to_string(), + "::1".to_string(), + ]; + if allow_any_host { + allowed_hosts.clear(); // rỗng = rmcp chấp nhận mọi Host header + } else { + allowed_hosts.extend(allow_host); + } + let use_root = root.as_str() != "/"; + if use_root && is_initialized(root) { + watcher::spawn(root.to_path_buf(), storage_dsn(root)); + } + return codegraph_mcp::serve_http(format, addr, allowed_hosts, enable_observability, api_key).await; + } if !mcp { - return Err(anyhow!("only --mcp transport supported")); + return Err(anyhow!( + "only --mcp (stdio) or --http (Streamable HTTP) supported" + )); } // MCP is session-driven: the agent binds a workspace at runtime via diff --git a/sql/README.md b/sql/README.md new file mode 100644 index 000000000..d8c172dd7 --- /dev/null +++ b/sql/README.md @@ -0,0 +1,140 @@ +# SQL schema design — storage shared (PostgreSQL / MySQL) + +Thiết kế schema cho **storage RDBMS mới** của codegraph, đặt **cạnh** các backend +local hiện có (sqlite / lmdb / redis / memory). Mục đích: 1 cơ sở dữ liệu dùng +**chung cho nhiều repository và nhiều server instance** — mỗi repo là một +partition độc lập, các instance (CLI / watcher / MCP stdio / MCP HTTP) cùng đọc +cùng ghi một DB. + +DuckDB / S3 lakehouse sẽ được thêm sau (`sql/duckdb/…`) trên **cùng model** +partition + sharding này. + +## Cấu trúc thư mục + +``` +sql/ + README.md ← file này + postgres/ ← DDL + migration PostgreSQL + 001-initial-schema.sql + mysql/ ← DDL + migration MySQL (cùng design, khác dialect) + 001-initial-schema.sql +``` + +## Quy ước đặt tên & quản lý version (migration) + +- Mỗi file schema đặt tên `NNN-.sql`, với `NNN` là số **3 chữ số + tăng dần** (`001-`, `002-`, ...). Tên mô tả ngắn gọn thay đổi (kebab-case), + VD `002-add-repo-statistics.sql`. +- **Thứ tự áp dụng = thứ tự số** (lexicographic). Migration chạy đúng thứ tự đó. +- **Không sửa / xoá file đã apply** — một thay đổi mới luôn là một file kế tiếp. + Nếu migration 002 cần sửa, viết 003 (ALTER/backfill), không sửa 002. +- Bảng `schema_migrations (version, applied_at)` (global, không có `repo_id`) + ghi lại version đã chạy — nền cho migration runner ở phase code + (sea-orm migrate / sqlx migrate đều theo convention này). +- **Migration là GLOBAL (schema-level)** — thay đổi cấu trúc bảng ảnh hưởng mọi + repo. Dữ liệu (`repo_id`) là runtime, không nằm trong file migration. + +## Mô hình dữ liệu + +### 1. Partition theo repository + +- Mọi bảng dữ liệu dẫn đầu bằng cột `repo_id VARCHAR(64) NOT NULL` — là **UUID** + sinh lúc `codegraph init`, lưu trong `.codegraph/config.toml` (`[storage] + repo_id = "…"`). Một project root (`.codegraph/`) = một repository. +- PK composite `(repo_id, …)` trên mọi bảng → các repo cô lập hoàn toàn; + re-index / xoá một repo chỉ là `DELETE … WHERE repo_id = ?`. +- `repo_id` nằm trong **handle của backend** (thuộc `Storage` impl), không đụng + trait `Storage`/`Tx` — mỗi `GraphIndex`/`SharedGraphIndex` instance = một repo. + +### 2. Sharding giữ nguyên + +- Radix trie (chain engine) vẫn dùng `CHAIN_SHARDING = 64`, + `shard_of(elem) = elem % 64` — toàn bộ key nằm trong đúng một shard (không + fan-out khi search). +- `rt_roots (repo_id, shard, root)` ánh xạ shard → root node; `rt_shortcuts` + (substring index) cũng theo `shard` như cũ. Sharding chỉ là partition nội bộ + của trie — không đổi hành vi query so với sqlite/lmdb hiện tại. + +### 3. Hai nhóm bảng + +**Entity store (`sg_*`)** — dữ liệu cấu trúc, dùng **cột thật** (lợi ích của +relational: query SQL trực tiếp, join, index; đồng thời sẵn sàng cho lakehouse / +parquet ở phase DuckDB): + +| Bảng | PK | Nội dung | +|---|---|---| +| `sg_symbols` | `(repo_id, id)` | `Symbol` — cột thật; `annotations` là cột JSON | +| `sg_files` | `(repo_id, path)` | `FileInfo` | +| `sg_call_records` | `(repo_id, func)` | call records của từng function (JSON bytes) | +| `sg_call_names` | `(repo_id, name)` | inverted index call name → call sites (JSON bytes) | +| `sg_meta` | `(repo_id)` | `version` của repo — dò freshness | +| `sg_next_id` | `(repo_id)` | registry counter (symbol id), seed `100` (`SYMBOL_BASE`) | + +**Radix trie (`rt_*`)** — dữ liệu nhị phân của trie (không có lợi ích relational, +giữ cột bytea/blob; vẫn partition theo `repo_id`): + +| Bảng | PK | Nội dung | +|---|---|---| +| `rt_nodes` | `(repo_id, id)` | node trie: `prefix` + `record`; id 0 = sentinel (EMPTY) | +| `rt_children` | `(repo_id, parent, child)` | cạnh cha-con | +| `rt_roots` | `(repo_id, shard)` | gốc từng shard (root 0 = EMPTY, tạo lazy) | +| `rt_meta` | `(repo_id, record)` | metadata opaque theo record | +| `rt_keylen` | `(repo_id, record)` | độ dài key (filter depth) | +| `rt_shortcuts` | `(repo_id, shard, elem, node_id)` | substring candidate index | +| `rt_chains` | `(repo_id, record)` | chain bytes (u64 LE/element) — nguồn rebuild | +| `rt_edges` / `rt_node_meta` | `(repo_id, …)` | legacy stream (trait còn giữ, GraphIndex chưa dùng) | +| `rt_node_blooms` | `(repo_id, id)` | bloom filter (feature `bloom-search`) | +| `rt_counter` | `(repo_id)` | node-id allocator, seed `1` | + +### 4. Pattern vận hành (comment chi tiết trong từng file) + +- **Seed per repo** (idempotent, `ON CONFLICT DO NOTHING` / `INSERT IGNORE`): + sentinel node 0, `rt_counter.next = 1`, `sg_next_id.next = 100`, `sg_meta.version = 0`. +- **Counter atomic per repo**: + - Postgres: `UPDATE rt_counter SET next = next + 1 WHERE repo_id = $1 RETURNING next - 1` + (tương tự `sg_next_id`). + - MySQL: `UPDATE rt_counter SET next = LAST_INSERT_ID(next) + 1 WHERE repo_id = ?` + rồi `SELECT LAST_INSERT_ID()` (connection-scoped, trả **next cũ** = id vừa cấp, + cùng semantics PG/sqlite — không dùng `LAST_INSERT_ID(next + 1)`, nó trả next mới). +- **Upsert**: Postgres `ON CONFLICT (repo_id, pk) DO UPDATE`; MySQL + `ON DUPLICATE KEY UPDATE`. +- **Probe version** (`SharedGraphIndex::ensure_fresh`): + `SELECT version FROM sg_meta WHERE repo_id = ?` — rẻ, độc lập với instance. +- **Full re-index** (clear): xoá toàn bộ `sg_*` + `rt_*` theo `repo_id`, reset + counters + version về seed. + +## Khác biệt giữa 2 dialect + +| | PostgreSQL | MySQL | +|---|---|---| +| DDL transactional | có (bọc `BEGIN/COMMIT`) | **không** — chạy tuần tự, không bọc transaction | +| binary | `BYTEA` | `LONGBLOB` | +| JSON | `JSONB` | `JSON` | +| timestamp | `TIMESTAMPTZ DEFAULT now()` | `TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP(6)` | +| cột key văn bản | `TEXT` thoải mái (PK được) | không cho `TEXT` làm PK/index toàn vẹn → `VARCHAR(700)` | +| case-sensitivity | chính xác theo byte | `COLLATE utf8mb4_bin` để giữ case-sensitive cho name/path | +| composite key `rt_shortcuts` | PK `(repo_id, shard, elem, node_id)` | `elem LONGBLOB` không vào PK được → index `elem(255)` prefix + PK không gồm elem (xem ghi chú) | + +> Ghi chú MySQL về giới hạn key: index key tối đa 3072 bytes (utf8mb4 → 768 ký +> tự). `repo_id VARCHAR(64)` + `name/file/path VARCHAR(700)` = 764 ký tự × 4 = +> 3056 bytes — vừa đủ. Giá trị dài hơn 700 ký tự cần hash key (md5/sha256) +> ở phase sau; schema hiện tại chấp nhận giới hạn này (tên call / path thực tế +> hiếm khi vượt). +> +> `rt_shortcuts.elem` là bytes nhị phân (element id encode) có thể rất dài → +> MySQL dùng prefix index `elem(255)`; Postgres giữ PK đầy đủ. Vì lookup luôn +> đi qua `(repo_id, shard, elem)` với elem truyền đúng độ dài thật, prefix index +> 255 bytes là đủ (kiểm tra lại khi implement — nếu cần chính xác tuyệt đối, +> thêm cột `elem_hash CHAR(32)`). + +## Liên hệ với code hiện tại & kế hoạch + +- Schema này là nguồn chân lý cho phase code: backend sea-orm (`RdbmsStorage` + implement `Storage` + `Tx`), routing DSN `postgres://`/`mysql://`, `repo_id` + vào config, `SharedGraphIndex` probe version, `IndexRegistry` (session giữ ref + tới index dùng chung). +- Mapping với `crates/codegraph-graph/src/storage/sqlite.rs` (schema hiện tại): + cùng tập bảng `sg_*`/`rt_*`, thêm cột `repo_id` + bỏ `CHECK(id = 1)` (đã thay + bằng PK `(repo_id)`), entity `sg_symbols` chuyển từ JSON BLOB sang cột thật. +- DuckDB / S3 lakehouse: `sql/duckdb/001-…sql` — cùng model, bảng thành file + parquet / duckdb, partition theo `repo_id`. diff --git a/sql/mysql/001-initial-schema.sql b/sql/mysql/001-initial-schema.sql new file mode 100644 index 000000000..3edbe651d --- /dev/null +++ b/sql/mysql/001-initial-schema.sql @@ -0,0 +1,239 @@ +-- ============================================================================= +-- codegraph-rs · storage migration 001 — initial schema (MySQL 8.0+) +-- ============================================================================= +-- Cùng design với `sql/postgres/001-initial-schema.sql` — chỉ khác dialect. +-- +-- LƯU Ý MySQL: +-- * DDL KHÔNG transactional — mỗi CREATE TABLE tự commit. Không bọc +-- BEGIN/COMMIT; chạy tuần tự theo thứ tự file. +-- * Không cho cột TEXT làm PRIMARY KEY / index toàn vẹn → mọi cột thuộc +-- khóa hoặc được index dùng `VARCHAR(700)` (đủ ngắn để nằm dưới giới hạn +-- index key 3072 bytes với utf8mb4, kể cả PK ghép với repo_id). Trường hợp +-- key dài hơn 700 ký tự → dùng hash key (md5/sha256) ở phase sau. +-- * `COLLATE utf8mb4_bin` giữ so sánh chính xác theo byte (name/path là key +-- case-sensitive; collation mặc định *_ci sẽ gộp 'Foo'/'foo'). +-- * id dùng BIGINT signed như sqlite hiện tại (u64 → i64, không đổi hành vi). +-- ============================================================================= + +-- ── Migration tracking (global) ────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS schema_migrations ( + version VARCHAR(64) NOT NULL PRIMARY KEY, + applied_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- ═══════════════════════════════════════════════════════════════════════════ +-- Entity store (sg_*) — partition theo repo_id +-- ═══════════════════════════════════════════════════════════════════════════ + +-- Symbol — tương ứng `codegraph_core::Symbol` (annotations = Vec). +CREATE TABLE IF NOT EXISTS sg_symbols ( + repo_id BIGINT NOT NULL, -- repository partition key (số u64) + id BIGINT NOT NULL, -- symbol id (registry global, ≥ 100) + name VARCHAR(700) NOT NULL DEFAULT '', + kind VARCHAR(32) NOT NULL DEFAULT '', -- SymbolKind: Function/Method/Class/... + scope VARCHAR(32) NOT NULL DEFAULT '', -- ScopeLevel: Global/ObjectField/Local/Parameter + scope_id BIGINT NOT NULL DEFAULT 0, -- id scope bao (0 = global) + type_ref BIGINT NOT NULL DEFAULT 0, -- id kiểu đã khai báo (0 = none) + type_name TEXT, -- raw type string, VD 'orderservice.OrderService' + file VARCHAR(700) NOT NULL DEFAULT '', + line INT NOT NULL DEFAULT 0, + end_line INT NOT NULL DEFAULT 0, + signature TEXT, + doc TEXT, + annotations JSON NOT NULL, -- app luôn ghi giá trị (JSON không cho DEFAULT) + language VARCHAR(64) NOT NULL DEFAULT '', + PRIMARY KEY (repo_id, id), + KEY idx_sg_symbols_repo_file (repo_id, file), + KEY idx_sg_symbols_repo_name (repo_id, name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- FileInfo — metadata file đã index. +-- `lines` được quote vì LINES là reserved word trong MySQL (LOAD DATA ... LINES). +CREATE TABLE IF NOT EXISTS sg_files ( + repo_id BIGINT NOT NULL, + path VARCHAR(700) NOT NULL, + language VARCHAR(64) NOT NULL DEFAULT '', + bytes BIGINT NOT NULL DEFAULT 0, + `lines` INT NOT NULL DEFAULT 0, + PRIMARY KEY (repo_id, path) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Call records của từng function — JSON bytes của `Vec`. +CREATE TABLE IF NOT EXISTS sg_call_records ( + repo_id BIGINT NOT NULL, + func BIGINT NOT NULL, -- caller symbol id + records LONGBLOB NOT NULL, -- serde_json bytes + PRIMARY KEY (repo_id, func) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Inverted index call name → call sites — JSON bytes của `Vec`. +CREATE TABLE IF NOT EXISTS sg_call_names ( + repo_id BIGINT NOT NULL, + name VARCHAR(700) NOT NULL, -- tên call (lowercase) + sites LONGBLOB NOT NULL, -- serde_json bytes + PRIMARY KEY (repo_id, name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Version index của repo — `SharedGraphIndex::ensure_fresh` probe ở đây. +CREATE TABLE IF NOT EXISTS sg_meta ( + repo_id BIGINT NOT NULL, + version BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (repo_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Registry counter — symbol id tiếp theo (SYMBOL_BASE = 100). +CREATE TABLE IF NOT EXISTS sg_next_id ( + repo_id BIGINT NOT NULL, + next BIGINT NOT NULL DEFAULT 100, -- SYMBOL_BASE + PRIMARY KEY (repo_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- ═══════════════════════════════════════════════════════════════════════════ +-- Radix trie (rt_*) — partition theo repo_id, giữ nguyên sharding element % 64 +-- ═══════════════════════════════════════════════════════════════════════════ + +-- Node của trie: prefix (bytes) + record (index key). id 0 = sentinel (EMPTY). +CREATE TABLE IF NOT EXISTS rt_nodes ( + repo_id BIGINT NOT NULL, + id BIGINT NOT NULL, + prefix LONGBLOB NOT NULL, + record BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (repo_id, id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Cạnh cha-con của trie. +CREATE TABLE IF NOT EXISTS rt_children ( + repo_id BIGINT NOT NULL, + parent BIGINT NOT NULL, + child BIGINT NOT NULL, + PRIMARY KEY (repo_id, parent, child), + KEY idx_rt_children_repo_parent (repo_id, parent) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Gốc mỗi shard: shard ∈ [0, 64). root = 0 nghĩa EMPTY — row tạo LAZY lần đầu +-- dùng shard (giống sqlite: get_root trả EMPTY khi thiếu row). +CREATE TABLE IF NOT EXISTS rt_roots ( + repo_id BIGINT NOT NULL, + shard INT NOT NULL, + root BIGINT NOT NULL DEFAULT 0, -- EMPTY = 0 + PRIMARY KEY (repo_id, shard) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Metadata opaque theo record (call-site info v.v.). +CREATE TABLE IF NOT EXISTS rt_meta ( + repo_id BIGINT NOT NULL, + record BIGINT NOT NULL, + meta LONGBLOB, + PRIMARY KEY (repo_id, record) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Độ dài key (số element) theo record — filter depth trong search. +CREATE TABLE IF NOT EXISTS rt_keylen ( + repo_id BIGINT NOT NULL, + record BIGINT NOT NULL, + len INT NOT NULL, + PRIMARY KEY (repo_id, record) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Shortcut index (substring search): node có prefix chứa elem → ứng viên KMP. +CREATE TABLE IF NOT EXISTS rt_shortcuts ( + repo_id BIGINT NOT NULL, + shard INT NOT NULL, + elem LONGBLOB NOT NULL, + node_id BIGINT NOT NULL, + PRIMARY KEY (repo_id, shard, elem(255), node_id), + KEY idx_rt_shortcuts_repo_shard_elem (repo_id, shard, elem(255)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Chain của function (record → bytes u64 LE mỗi element). Nguồn chân lý để +-- rebuild engine khi reopen (`GraphIndex::rebuild` → `all_chains()`). +CREATE TABLE IF NOT EXISTS rt_chains ( + repo_id BIGINT NOT NULL, + record BIGINT NOT NULL, -- func id + chain LONGBLOB NOT NULL, + PRIMARY KEY (repo_id, record) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Edge data stream (legacy — Storage trait còn giữ, GraphIndex chưa dùng). +CREATE TABLE IF NOT EXISTS rt_edges ( + repo_id BIGINT NOT NULL, + id BIGINT NOT NULL, + data LONGBLOB, + PRIMARY KEY (repo_id, id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Node metadata stream (Node JSON theo element — legacy, chưa dùng). +CREATE TABLE IF NOT EXISTS rt_node_meta ( + repo_id BIGINT NOT NULL, + elem BIGINT NOT NULL, + meta LONGBLOB, + PRIMARY KEY (repo_id, elem) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Bloom filter per node (feature `bloom-search`). +CREATE TABLE IF NOT EXISTS rt_node_blooms ( + repo_id BIGINT NOT NULL, + id BIGINT NOT NULL, + bloom LONGBLOB, + PRIMARY KEY (repo_id, id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Node-id allocator (per repo — các shard dùng chung một dãy id như sqlite). +CREATE TABLE IF NOT EXISTS rt_counter ( + repo_id BIGINT NOT NULL, + next BIGINT NOT NULL DEFAULT 1, + PRIMARY KEY (repo_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- ═══════════════════════════════════════════════════════════════════════════ +-- Pattern dùng chung (thực thi ở tầng storage — KHÔNG nằm trong migration, +-- vì repo_id là dữ liệu runtime từ config) +-- ═══════════════════════════════════════════════════════════════════════════ +-- +-- [Seed per repo] — lần đầu chạm repo, upsert idempotent (repo_id = số u64): +-- INSERT IGNORE INTO rt_nodes (repo_id, id, prefix, record) VALUES (?, 0, '', 0); +-- INSERT IGNORE INTO rt_counter (repo_id, next) VALUES (?, 1); +-- INSERT IGNORE INTO sg_next_id (repo_id, next) VALUES (?, 100); +-- INSERT IGNORE INTO sg_meta (repo_id, version) VALUES (?, 0); +-- +-- [Node id alloc] — atomic, per repo. `LAST_INSERT_ID(expr)` là connection-scoped; +-- idiom dưới giữ semantics GIỐNG PG/sqlite: id cấp = next cũ, rồi next += 1. +-- (KHÔNG dùng `LAST_INSERT_ID(next + 1)` — nó trả next MỚI, sai id vừa cấp.) +-- UPDATE rt_counter SET next = LAST_INSERT_ID(next) + 1 WHERE repo_id = ?; +-- SELECT LAST_INSERT_ID(); -- = next cũ (id vừa cấp) +-- +-- [Symbol registry id alloc]: +-- UPDATE sg_next_id SET next = LAST_INSERT_ID(next) + 1 WHERE repo_id = ?; +-- SELECT LAST_INSERT_ID(); +-- +-- [Upsert entity] — ví dụ sg_symbols: +-- INSERT INTO sg_symbols (repo_id, id, name, kind, scope, scope_id, type_ref, +-- type_name, file, line, end_line, signature, doc, +-- annotations, language) +-- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +-- ON DUPLICATE KEY UPDATE +-- name = VALUES(name), kind = VALUES(kind), ..., +-- annotations = VALUES(annotations), language = VALUES(language); +-- +-- [Probe version] — `SharedGraphIndex::current_version`: +-- SELECT version FROM sg_meta WHERE repo_id = ?; +-- +-- [Full re-index (clear)] — xoá toàn bộ data repo rồi ingest lại: +-- DELETE FROM sg_symbols WHERE repo_id = ?; +-- DELETE FROM sg_files WHERE repo_id = ?; +-- DELETE FROM sg_call_records WHERE repo_id = ?; +-- DELETE FROM sg_call_names WHERE repo_id = ?; +-- DELETE FROM rt_nodes WHERE repo_id = ?; +-- DELETE FROM rt_children WHERE repo_id = ?; +-- DELETE FROM rt_roots WHERE repo_id = ?; +-- DELETE FROM rt_meta WHERE repo_id = ?; +-- DELETE FROM rt_keylen WHERE repo_id = ?; +-- DELETE FROM rt_shortcuts WHERE repo_id = ?; +-- DELETE FROM rt_chains WHERE repo_id = ?; +-- DELETE FROM rt_edges WHERE repo_id = ?; +-- DELETE FROM rt_node_meta WHERE repo_id = ?; +-- DELETE FROM rt_node_blooms WHERE repo_id = ?; +-- UPDATE rt_counter SET next = 1 WHERE repo_id = ?; +-- UPDATE sg_next_id SET next = 100 WHERE repo_id = ?; +-- UPDATE sg_meta SET version = 0 WHERE repo_id = ?; +-- ═══════════════════════════════════════════════════════════════════════════ diff --git a/sql/mysql/002-add-repos-registry.sql b/sql/mysql/002-add-repos-registry.sql new file mode 100644 index 000000000..fc12a464f --- /dev/null +++ b/sql/mysql/002-add-repos-registry.sql @@ -0,0 +1,42 @@ +-- ============================================================================= +-- codegraph-rs · storage migration 002 — repos registry (global mapping) (MySQL) +-- ============================================================================= +-- Cùng design với `sql/postgres/002-add-repos-registry.sql` — chỉ khác dialect. +-- +-- Bảng mapping repo_id → shard — phần "quản lý mapping" của thiết kế sharding. +-- GLOBAL: KHÔNG partition theo repo_id, và được NHÂN BẢN trên MỌI shard server +-- (mỗi shard giữ bản sao đầy đủ) — bất kỳ instance nào cũng tra được repo thuộc +-- shard nào mà không cần biết trước điểm tra. +-- +-- Vai trò: +-- * `shard` = chỉ mục vào `dsns` của shard server ĐƯỢC GÁN. Gán ĐÚNG MỘT LẦN +-- lúc đăng ký (lần chạm DB đầu tiên), mọi open sau ĐỌC từ bảng này — KHÔNG +-- recompute `repo_id % N`. Đổi số lượng DSN không làm repo dịch server +-- (dữ liệu không mất; chỉ repo MỚI tính theo N mới). +-- * `root` = root path chuẩn → lookup ngược: cùng root path (clone/máy khác) +-- nhận CÙNG repo_id → dùng chung partition trong DB. +-- +-- LƯU Ý DIALECT: +-- * MySQL không hỗ trợ `CREATE INDEX IF NOT EXISTS` — an toàn vì migration +-- runner chạy MỘT LẦN per server (track theo `schema_migrations`). +-- * root dùng VARCHAR(700) (giới hạn index key utf8mb4 3072 bytes — xem 001). +-- +-- Quy trình (thực thi ở tầng storage/repo resolver — KHÔNG nằm trong migration): +-- [Lookup by repo_id] — đã biết repo_id (config): đọc ở shard `repo_id % N`: +-- SELECT shard FROM repos WHERE repo_id = ?; +-- [Adopt by root] — config thiếu repo_id: đọc ở bất kỳ shard (chuẩn: shard 0): +-- SELECT repo_id, shard FROM repos WHERE root = ?; +-- [Register] — repo mới: gán shard = repo_id % N, ghi vào MỌI shard (idempotent): +-- INSERT INTO repos (repo_id, shard, root) VALUES (?, ?, ?) +-- ON DUPLICATE KEY UPDATE repo_id = repo_id; -- no-op nếu đã tồn tại +-- -- lặp lại cho từng shard server +-- ============================================================================= + +CREATE TABLE IF NOT EXISTS repos ( + repo_id BIGINT NOT NULL PRIMARY KEY, -- repo_id (số u64, random lúc init) + shard INT NOT NULL, -- shard server được gán (index vào dsns) + root VARCHAR(700) NOT NULL, -- root path chuẩn để lookup + created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +-- Lookup theo root (adopt cùng repo_id cho clone/máy khác). +CREATE INDEX idx_repos_root ON repos (root); \ No newline at end of file diff --git a/sql/postgres/001-initial-schema.sql b/sql/postgres/001-initial-schema.sql new file mode 100644 index 000000000..2e2b62ee8 --- /dev/null +++ b/sql/postgres/001-initial-schema.sql @@ -0,0 +1,248 @@ +-- ============================================================================= +-- codegraph-rs · storage migration 001 — initial schema (PostgreSQL) +-- ============================================================================= +-- Quy ước migration: thư mục `sql/postgres/`, mỗi file đặt tên +-- `NNN-.sql` (001-, 002-, ...). Áp dụng theo thứ tự số; KHÔNG sửa +-- file đã apply — thay đổi mới phải là file kế tiếp. Bảng `schema_migrations` +-- ghi lại version đã chạy (nền cho migration runner ở phase code). +-- +-- Thiết kế (chi tiết xem sql/README.md): +-- * Mọi bảng dữ liệu dẫn đầu bằng `repo_id` (SỐ u64, sinh ngẫu nhiên lúc +-- `codegraph init`, lưu `.codegraph/config.toml`) — 1 repository = 1 partition. +-- PK composite `(repo_id, ...)`. Re-index / xoá repo = `DELETE WHERE repo_id = ?`. +-- Shard server = `repo_id % số_lượng_dsn` (xem mục sharding trong README). +-- * `sg_*` = entity store (cột thật — Symbol/FileInfo/CallRecord/CallSite, +-- phục vụ query SQL trực tiếp + sẵn sàng cho lakehouse/parquet). +-- * `rt_*` = radix trie (dữ liệu nhị phân — prefix/chain/shortcut/meta), +-- giữ nguyên cơ chế sharding hiện tại (CHAIN_SHARDING = 64, +-- `shard_of(elem) = elem % 64`). Không có lợi ích relational nên giữ cột +-- bytea; vẫn partition theo repo_id như mọi bảng khác. +-- * Migration (schema) là GLOBAL — không partition theo repo. +-- ============================================================================= + +BEGIN; + +-- ── Migration tracking (global) ────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS schema_migrations ( + version VARCHAR(64) NOT NULL PRIMARY KEY, -- tên file, VD '001-initial-schema' + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- ═══════════════════════════════════════════════════════════════════════════ +-- Entity store (sg_*) — partition theo repo_id +-- ═══════════════════════════════════════════════════════════════════════════ + +-- Symbol — tương ứng `codegraph_core::Symbol` (annotations = Vec). +CREATE TABLE IF NOT EXISTS sg_symbols ( + repo_id BIGINT NOT NULL, -- repository partition key (số u64) + id BIGINT NOT NULL, -- symbol id (registry global, ≥ 100) + name TEXT NOT NULL DEFAULT '', + kind VARCHAR(32) NOT NULL DEFAULT '', -- SymbolKind: Function/Method/Class/... + scope VARCHAR(32) NOT NULL DEFAULT '', -- ScopeLevel: Global/ObjectField/Local/Parameter + scope_id BIGINT NOT NULL DEFAULT 0, -- id scope bao (0 = global) + type_ref BIGINT NOT NULL DEFAULT 0, -- id kiểu đã khai báo (0 = none) + type_name TEXT, -- raw type string, VD 'orderservice.OrderService' + file TEXT NOT NULL DEFAULT '', + line INTEGER NOT NULL DEFAULT 0, + end_line INTEGER NOT NULL DEFAULT 0, + signature TEXT, + doc TEXT, + annotations JSONB NOT NULL DEFAULT '[]'::jsonb, + language TEXT NOT NULL DEFAULT '', + PRIMARY KEY (repo_id, id) +); +-- File filter (tool list theo root/file) + name lookup convenience. +CREATE INDEX IF NOT EXISTS idx_sg_symbols_repo_file ON sg_symbols (repo_id, file); +CREATE INDEX IF NOT EXISTS idx_sg_symbols_repo_name ON sg_symbols (repo_id, name); + +-- FileInfo — metadata file đã index. +CREATE TABLE IF NOT EXISTS sg_files ( + repo_id BIGINT NOT NULL, + path TEXT NOT NULL, + language TEXT NOT NULL DEFAULT '', + bytes BIGINT NOT NULL DEFAULT 0, + lines INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (repo_id, path) +); + +-- Call records của từng function — JSON bytes của `Vec`. +CREATE TABLE IF NOT EXISTS sg_call_records ( + repo_id BIGINT NOT NULL, + func BIGINT NOT NULL, -- caller symbol id + records BYTEA NOT NULL, -- serde_json bytes + PRIMARY KEY (repo_id, func) +); + +-- Inverted index call name → call sites — JSON bytes của `Vec`. +CREATE TABLE IF NOT EXISTS sg_call_names ( + repo_id BIGINT NOT NULL, + name TEXT NOT NULL, -- tên call (lowercase) + sites BYTEA NOT NULL, -- serde_json bytes + PRIMARY KEY (repo_id, name) +); + +-- Version index của repo — `SharedGraphIndex::ensure_fresh` probe ở đây +-- (mỗi full re-index bump version → snapshot in-memory cũ thấy stale, rebuild). +CREATE TABLE IF NOT EXISTS sg_meta ( + repo_id BIGINT NOT NULL, + version BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (repo_id) +); + +-- Registry counter — symbol id tiếp theo (SYMBOL_BASE = 100). +CREATE TABLE IF NOT EXISTS sg_next_id ( + repo_id BIGINT NOT NULL, + next BIGINT NOT NULL DEFAULT 100, -- SYMBOL_BASE + PRIMARY KEY (repo_id) +); + +-- ═══════════════════════════════════════════════════════════════════════════ +-- Radix trie (rt_*) — partition theo repo_id, giữ nguyên sharding element % 64 +-- ═══════════════════════════════════════════════════════════════════════════ + +-- Node của trie: prefix (bytes) + record (index key). id 0 = sentinel (EMPTY). +CREATE TABLE IF NOT EXISTS rt_nodes ( + repo_id BIGINT NOT NULL, + id BIGINT NOT NULL, + prefix BYTEA NOT NULL, + record BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (repo_id, id) +); + +-- Cạnh cha-con của trie. +CREATE TABLE IF NOT EXISTS rt_children ( + repo_id BIGINT NOT NULL, + parent BIGINT NOT NULL, + child BIGINT NOT NULL, + PRIMARY KEY (repo_id, parent, child) +); +-- Truy vấn children của một node — theo (repo_id, parent). +CREATE INDEX IF NOT EXISTS idx_rt_children_repo_parent ON rt_children (repo_id, parent); + +-- Gốc mỗi shard: shard ∈ [0, 64). root = 0 nghĩa EMPTY — row tạo LAZY lần đầu +-- dùng shard (giống sqlite: get_root trả EMPTY khi thiếu row). +CREATE TABLE IF NOT EXISTS rt_roots ( + repo_id BIGINT NOT NULL, + shard INTEGER NOT NULL, + root BIGINT NOT NULL DEFAULT 0, -- EMPTY = 0 + PRIMARY KEY (repo_id, shard) +); + +-- Metadata opaque theo record (call-site info v.v.). +CREATE TABLE IF NOT EXISTS rt_meta ( + repo_id BIGINT NOT NULL, + record BIGINT NOT NULL, + meta BYTEA, + PRIMARY KEY (repo_id, record) +); + +-- Độ dài key (số element) theo record — filter depth trong search. +CREATE TABLE IF NOT EXISTS rt_keylen ( + repo_id BIGINT NOT NULL, + record BIGINT NOT NULL, + len INTEGER NOT NULL, + PRIMARY KEY (repo_id, record) +); + +-- Shortcut index (substring search): node có prefix chứa elem → ứng viên KMP. +CREATE TABLE IF NOT EXISTS rt_shortcuts ( + repo_id BIGINT NOT NULL, + shard INTEGER NOT NULL, + elem BYTEA NOT NULL, + node_id BIGINT NOT NULL, + PRIMARY KEY (repo_id, shard, elem, node_id) +); +-- Lookup: tập node id chứa elem trong một shard. +CREATE INDEX IF NOT EXISTS idx_rt_shortcuts_repo_shard_elem + ON rt_shortcuts (repo_id, shard, elem); + +-- Chain của function (record → bytes u64 LE mỗi element). Nguồn chân lý để +-- rebuild engine khi reopen (`GraphIndex::rebuild` → `all_chains()`). +CREATE TABLE IF NOT EXISTS rt_chains ( + repo_id BIGINT NOT NULL, + record BIGINT NOT NULL, -- func id + chain BYTEA NOT NULL, + PRIMARY KEY (repo_id, record) +); + +-- Edge data stream (legacy — Storage trait còn giữ, GraphIndex chưa dùng). +CREATE TABLE IF NOT EXISTS rt_edges ( + repo_id BIGINT NOT NULL, + id BIGINT NOT NULL, + data BYTEA, + PRIMARY KEY (repo_id, id) +); + +-- Node metadata stream (Node JSON theo element — legacy, chưa dùng). +CREATE TABLE IF NOT EXISTS rt_node_meta ( + repo_id BIGINT NOT NULL, + elem BIGINT NOT NULL, + meta BYTEA, + PRIMARY KEY (repo_id, elem) +); + +-- Bloom filter per node (feature `bloom-search`). +CREATE TABLE IF NOT EXISTS rt_node_blooms ( + repo_id BIGINT NOT NULL, + id BIGINT NOT NULL, + bloom BYTEA, + PRIMARY KEY (repo_id, id) +); + +-- Node-id allocator (per repo — các shard dùng chung một dãy id như sqlite). +CREATE TABLE IF NOT EXISTS rt_counter ( + repo_id BIGINT NOT NULL, + next BIGINT NOT NULL DEFAULT 1, + PRIMARY KEY (repo_id) +); + +-- ═══════════════════════════════════════════════════════════════════════════ +-- Pattern dùng chung (thực thi ở tầng storage — KHÔNG nằm trong migration, +-- vì repo_id là dữ liệu runtime từ config) +-- ═══════════════════════════════════════════════════════════════════════════ +-- +-- [Seed per repo] — lần đầu chạm repo, upsert idempotent (repo_id = số u64): +-- INSERT INTO rt_nodes (repo_id, id, prefix, record) VALUES ($1, 0, '', 0) +-- ON CONFLICT DO NOTHING; +-- INSERT INTO rt_counter (repo_id, next) VALUES ($1, 1) ON CONFLICT DO NOTHING; +-- INSERT INTO sg_next_id (repo_id, next) VALUES ($1, 100) ON CONFLICT DO NOTHING; +-- INSERT INTO sg_meta (repo_id, version) VALUES ($1, 0) ON CONFLICT DO NOTHING; +-- +-- [Node id alloc] — atomic, per repo: +-- UPDATE rt_counter SET next = next + 1 WHERE repo_id = $1 RETURNING next - 1; +-- +-- [Symbol registry id alloc]: +-- UPDATE sg_next_id SET next = next + 1 WHERE repo_id = $1 RETURNING next - 1; +-- +-- [Upsert entity] — ví dụ sg_symbols: +-- INSERT INTO sg_symbols (repo_id, id, name, kind, scope, scope_id, type_ref, +-- type_name, file, line, end_line, signature, doc, +-- annotations, language) +-- VALUES ($1, ..., $15) +-- ON CONFLICT (repo_id, id) DO UPDATE SET +-- name = $3, kind = $4, ..., annotations = $14, language = $15; +-- +-- [Probe version] — `SharedGraphIndex::current_version`: +-- SELECT version FROM sg_meta WHERE repo_id = $1; +-- +-- [Full re-index (clear)] — xoá toàn bộ data repo rồi ingest lại: +-- DELETE FROM sg_symbols WHERE repo_id = $1; +-- DELETE FROM sg_files WHERE repo_id = $1; +-- DELETE FROM sg_call_records WHERE repo_id = $1; +-- DELETE FROM sg_call_names WHERE repo_id = $1; +-- DELETE FROM rt_nodes WHERE repo_id = $1; +-- DELETE FROM rt_children WHERE repo_id = $1; +-- DELETE FROM rt_roots WHERE repo_id = $1; +-- DELETE FROM rt_meta WHERE repo_id = $1; +-- DELETE FROM rt_keylen WHERE repo_id = $1; +-- DELETE FROM rt_shortcuts WHERE repo_id = $1; +-- DELETE FROM rt_chains WHERE repo_id = $1; +-- DELETE FROM rt_edges WHERE repo_id = $1; +-- DELETE FROM rt_node_meta WHERE repo_id = $1; +-- DELETE FROM rt_node_blooms WHERE repo_id = $1; +-- UPDATE rt_counter SET next = 1 WHERE repo_id = $1; +-- UPDATE sg_next_id SET next = 100 WHERE repo_id = $1; +-- UPDATE sg_meta SET version = 0 WHERE repo_id = $1; +-- ═══════════════════════════════════════════════════════════════════════════ + +COMMIT; diff --git a/sql/postgres/002-add-repos-registry.sql b/sql/postgres/002-add-repos-registry.sql new file mode 100644 index 000000000..4d1de7840 --- /dev/null +++ b/sql/postgres/002-add-repos-registry.sql @@ -0,0 +1,39 @@ +-- ============================================================================= +-- codegraph-rs · storage migration 002 — repos registry (global mapping) +-- ============================================================================= +-- Bảng mapping repo_id → shard — phần "quản lý mapping" của thiết kế sharding. +-- GLOBAL: KHÔNG partition theo repo_id, và được NHÂN BẢN trên MỌI shard server +-- (mỗi shard giữ bản sao đầy đủ) — bất kỳ instance nào cũng tra được repo thuộc +-- shard nào mà không cần biết trước điểm tra. +-- +-- Vai trò: +-- * `shard` = chỉ mục vào `dsns` của shard server ĐƯỢC GÁN. Gán ĐÚNG MỘT LẦN +-- lúc đăng ký (lần chạm DB đầu tiên), mọi open sau ĐỌC từ bảng này — KHÔNG +-- recompute `repo_id % N`. Đổi số lượng DSN không làm repo dịch server +-- (dữ liệu không mất; chỉ repo MỚI tính theo N mới). +-- * `root` = root path chuẩn → lookup ngược: cùng root path (clone/máy khác) +-- nhận CÙNG repo_id → dùng chung partition trong DB. +-- +-- Quy trình (thực thi ở tầng storage/repo resolver — KHÔNG nằm trong migration): +-- [Lookup by repo_id] — đã biết repo_id (config): đọc ở shard `repo_id % N`, +-- mapping nhân bản nên tìm được dù N đã đổi: +-- SELECT shard FROM repos WHERE repo_id = ?; +-- [Adopt by root] — config thiếu repo_id: đọc ở bất kỳ shard (chuẩn: shard 0): +-- SELECT repo_id, shard FROM repos WHERE root = ?; +-- [Register] — repo mới: gán shard = repo_id % N, ghi vào MỌI shard (idempotent): +-- INSERT INTO repos (repo_id, shard, root) VALUES (?, ?, ?) +-- ON CONFLICT (repo_id) DO NOTHING; -- lặp lại cho từng shard server +-- ============================================================================= + +BEGIN; + +CREATE TABLE IF NOT EXISTS repos ( + repo_id BIGINT NOT NULL PRIMARY KEY, -- repo_id (số u64, random lúc init) + shard INT NOT NULL, -- shard server được gán (index vào dsns) + root TEXT NOT NULL, -- root path chuẩn để lookup + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +-- Lookup theo root (adopt cùng repo_id cho clone/máy khác). +CREATE INDEX IF NOT EXISTS idx_repos_root ON repos (root); + +COMMIT; \ No newline at end of file From b6c6b7eb758bcb299af02514510eef750cc12de0 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 15 Aug 2026 16:46:39 +0700 Subject: [PATCH 2/9] Setup unit-tests to new storages --- .github/workflows/integration.yml | 121 +++ README.md | 55 + crates/codegraph-core/src/route.rs | 8 + crates/codegraph-core/src/semgraph.rs | 11 + crates/codegraph-extract/src/config.rs | 103 +- crates/codegraph-graph/src/lib.rs | 121 ++- crates/codegraph-graph/src/shared.rs | 152 ++- crates/codegraph-graph/src/storage/mysql.rs | 857 ++++++++++++++-- .../codegraph-graph/src/storage/postgres.rs | 945 +++++++++++++++--- crates/codegraph-graph/tests/rdbms.rs | 163 +++ crates/codegraph-graph/tests/redis.rs | 147 +++ crates/codegraph-mcp/Cargo.toml | 3 + crates/codegraph-mcp/src/session.rs | 48 +- crates/codegraph/Cargo.toml | 5 +- crates/codegraph/src/main.rs | 13 +- sql/README.md | 17 +- 16 files changed, 2447 insertions(+), 322 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 crates/codegraph-graph/tests/rdbms.rs create mode 100644 crates/codegraph-graph/tests/redis.rs diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 000000000..745e95504 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,121 @@ +name: Integration (Postgres / MySQL / Redis) + +# Chạy test tích hợp trên backend thật (Postgres/MySQL/Redis) qua service +# container của GitHub Actions. Schema được apply thủ công (`sql//*`) +# trước khi chạy test — khớp thiết kế "migration thủ công" của repo. +# +# Test trong `tests/rdbms.rs` / `tests/redis.rs` bị `#[ignore]` và chỉ chạy khi +# có DSN tương ứng → không ảnh hưởng `cargo test` thường (CI chính ở ci.yml). +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + clippy-gated: + name: clippy (rdbms + redis test targets) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + # Đảm bảo các file test gated (tests/rdbms.rs, tests/redis.rs) vẫn + # clippy-sạch dù CI chính chỉ build với default features. + - run: cargo clippy -p codegraph-graph --features postgres,mysql,redis --tests -- -D warnings + + postgres: + name: postgres + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: codegraph + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + TEST_RDBMS_DSN: "postgres://postgres:postgres@127.0.0.1:5432/codegraph" + TEST_RDBMS_REPO_ID: "1" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install postgresql-client + run: sudo apt-get update && sudo apt-get install -y postgresql-client + - name: Apply schema (manual migration) + run: | + psql "$TEST_RDBMS_DSN" -f sql/postgres/001-initial-schema.sql + psql "$TEST_RDBMS_DSN" -f sql/postgres/002-add-repos-registry.sql + - name: Run integration tests + run: cargo test -p codegraph-graph --features postgres --test rdbms -- --ignored --nocapture + + mysql: + name: mysql + runs-on: ubuntu-latest + services: + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: postgres + MYSQL_DATABASE: codegraph + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root -ppostgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + TEST_RDBMS_DSN: "mysql://root:postgres@127.0.0.1:3306/codegraph" + TEST_RDBMS_REPO_ID: "1" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install mysql-client + run: sudo apt-get update && sudo apt-get install -y mysql-client + - name: Apply schema (manual migration) + run: | + mysql -h 127.0.0.1 -P 3306 -u root -ppostgres codegraph < sql/mysql/001-initial-schema.sql + mysql -h 127.0.0.1 -P 3306 -u root -ppostgres codegraph < sql/mysql/002-add-repos-registry.sql + - name: Run integration tests + run: cargo test -p codegraph-graph --features mysql --test rdbms -- --ignored --nocapture + + redis: + name: redis + runs-on: ubuntu-latest + services: + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + TEST_REDIS_DSN: "redis://127.0.0.1:6379" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + # Unit test nội bộ (storage/redis.rs) chạy trên DB 15. + - name: Run storage unit tests + run: cargo test -p codegraph-graph --features redis + # Integration test (GraphIndex roundtrip) chạy trên DB 0 (DSN mặc định). + - name: Run integration tests + run: cargo test -p codegraph-graph --features redis --test redis -- --ignored --nocapture diff --git a/README.md b/README.md index 03c14db1f..50716daf3 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,54 @@ exclude = [ ] ``` +### Postgres / MySQL (multi-tenant, sharded) + +CodeGraph can store the index in PostgreSQL or MySQL instead of the local +SQLite file. Every table is partitioned by a leading `repo_id` (a `u64` +partition key), so each project root (`.codegraph/`) maps to its own +partition — re-indexing or deleting one repo never touches another. Sharding +is `repo_id % N` across the configured DSN list. + +Build with the `rdbms` feature (it is **on by default** for the `codegraph` +binary): + +```bash +cargo build --features rdbms # default for `codegraph` +cargo build -p codegraph-mcp --features rdbms +``` + +`.codegraph/config.toml`: + +```toml +[storage] +type = "postgres" +# type = "mysql" +# Shard DSNs — shard = repo_id % len(dsns). One entry = single shard. +dsns = [ + "postgres://user:pass@db1:5432/codegraph", + "postgres://user:pass@db2:5432/codegraph", +] +# repo_id is generated automatically by `codegraph init` (self-heal) and +# written here. Do not edit it by hand. +# repo_id = 14028493579208694412 +``` + +**Schema is applied manually** — the binary does not run migrations. Run the +SQL files from `sql//` in order (currently `001-initial-schema.sql` +and `002-add-repos-registry.sql`) against every shard server before indexing: + +```bash +psql "$DSN" -f sql/postgres/001-initial-schema.sql +psql "$DSN" -f sql/postgres/002-add-repos-registry.sql +# mysql: +# mysql "$DB" < sql/mysql/001-initial-schema.sql +# mysql "$DB" < sql/mysql/002-add-repos-registry.sql +``` + +Then `codegraph init` (CLI) or `codegraph_init` (MCP tool) generates the +`repo_id` and stores the index on the right shard automatically. See +`sql/README.md` for the full multi-tenant + sharding design. + ### C vs C++ headers (`.h`) By default, `.h` files are resolved automatically: @@ -344,11 +392,18 @@ cargo test -p codegraph-extract --features lang-python Feature flags on `codegraph-graph`: - `sqlite` — sqlite storage backend (enabled on `codegraph`, `codegraph-mcp`, `codegraph-viz`) - `redis` — redis storage backend (compile-only verify, runtime needs server) +- `postgres` — PostgreSQL storage backend (multi-tenant, sharded) +- `mysql` — MySQL storage backend (multi-tenant, sharded) + +The `codegraph` and `codegraph-mcp` binaries expose a convenience `rdbms` +feature that turns on both `postgres` and `mysql` (it is **on by default** +for `codegraph`): ```sh # Full feature verification cargo check --workspace --features sqlite cargo check -p codegraph-graph --features redis +cargo check -p codegraph --features rdbms ``` ## License diff --git a/crates/codegraph-core/src/route.rs b/crates/codegraph-core/src/route.rs index 153653e5e..6285add31 100644 --- a/crates/codegraph-core/src/route.rs +++ b/crates/codegraph-core/src/route.rs @@ -56,5 +56,13 @@ impl StorageRoute { _ => None, } } + + /// Root path chuẩn hiện có trong route (`None` nếu không phải `Sharded`). + pub fn root(&self) -> Option<&str> { + match self { + StorageRoute::Sharded { root, .. } => root.as_deref(), + _ => None, + } + } } diff --git a/crates/codegraph-core/src/semgraph.rs b/crates/codegraph-core/src/semgraph.rs index 68cff5154..3bc6a0d3d 100644 --- a/crates/codegraph-core/src/semgraph.rs +++ b/crates/codegraph-core/src/semgraph.rs @@ -183,6 +183,17 @@ impl ScopeLevel { Self::Parameter => "parameter", } } + + /// Parse từ chuỗi (`as_str()` ngược lại) — `None` nếu không khớp. + pub fn parse(s: &str) -> Option { + Some(match s { + "global" => Self::Global, + "object_field" => Self::ObjectField, + "local" => Self::Local, + "parameter" => Self::Parameter, + _ => return None, + }) + } } /// Phân loại tác động bên ngoài của một call (để impact/report). diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index 59eb25746..a3386d92e 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -1,7 +1,7 @@ use crate::languages::effects::EffectClassifier; use crate::project::{project_db_path, project_dir}; use camino::Utf8Path; -use codegraph_core::{EffectCallPattern, EffectRule, EffectType}; +use codegraph_core::{EffectCallPattern, EffectRule, EffectType, StorageRoute}; use serde::Deserialize; use std::fs; @@ -27,6 +27,10 @@ pub enum StorageKind { Redis, /// In-memory — không persist. Memory, + /// PostgreSQL — multi-tenant, partition theo `repo_id`. + Postgres, + /// MySQL — multi-tenant, partition theo `repo_id`. + MySql, } impl StorageKind { @@ -35,9 +39,16 @@ impl StorageKind { "lmdb" => StorageKind::Lmdb, "redis" => StorageKind::Redis, "memory" | "in-memory" | "in_memory" => StorageKind::Memory, + "postgres" | "postgresql" | "pg" => StorageKind::Postgres, + "mysql" | "maria" | "mariadb" => StorageKind::MySql, _ => StorageKind::Sqlite, } } + + /// Backend này có phải RDBMS (Postgres/MySQL) hay không. + pub fn is_rdbms(self) -> bool { + matches!(self, StorageKind::Postgres | StorageKind::MySql) + } } #[derive(Debug, Default, Deserialize)] @@ -54,12 +65,19 @@ struct ConfigFile { #[derive(Debug, Default, Deserialize)] struct StorageSection { - /// `"sqlite"`, `"lmdb"`, `"redis"`, `"memory"`. + /// `"sqlite"`, `"lmdb"`, `"redis"`, `"memory"`, `"postgres"`, `"mysql"`. #[serde(default, rename = "type")] type_: Option, /// DSN override — ví dụ `lmdb:///data/codegraph.db`. #[serde(default)] dsn: Option, + /// `repo_id` (u64) dùng làm partition key cho backend RDBMS + /// (Postgres/MySQL, multi-tenant). Tự sinh bởi `codegraph init` nếu thiếu. + #[serde(default)] + repo_id: Option, + /// Danh sách DSN shard cho backend RDBMS. Shard = `repo_id % len(dsns)`. + #[serde(default)] + dsns: Vec, } #[derive(Debug, Default, Deserialize)] @@ -94,6 +112,11 @@ pub struct StorageConfig { pub kind: StorageKind, /// DSN override (`None` = dựng từ `kind` + project path). pub dsn: Option, + /// `repo_id` (u64) — partition key cho backend RDBMS. `None` nếu chưa sinh + /// (chỉ hợp lệ khi `kind` không phải RDBMS). + pub repo_id: Option, + /// Danh sách DSN shard cho backend RDBMS (shard = `repo_id % len`). + pub dsns: Vec, } impl ExtractConfig { @@ -120,6 +143,8 @@ impl ExtractConfig { .map(StorageKind::parse) .unwrap_or_default(), dsn: file.storage.dsn, + repo_id: file.storage.repo_id, + dsns: file.storage.dsns, }, } } @@ -141,7 +166,76 @@ impl ExtractConfig { StorageKind::Lmdb => Some(format!("lmdb://{}", project_dir(root).join("db.lmdb"))), StorageKind::Redis => None, StorageKind::Memory => None, + StorageKind::Postgres | StorageKind::MySql => None, + } + } + + /// `StorageRoute` mô tả cách mở index — thay thế cho `storage_dsn` khi + /// backend có thể là RDBMS (multi-tenant + sharding). + /// + /// - `memory` → `Memory` + /// - `sqlite` / `lmdb` / `redis` → `Local(dsn)` + /// - `postgres` / `mysql` → `Sharded { dsns, repo_id, root }` + /// (`repo_id` phải đã được sinh bởi `ensure_repo_id`; nếu thiếu → `None`) + pub fn storage_route(&self, root: &Utf8Path) -> Option { + match self.storage.kind { + StorageKind::Memory => Some(StorageRoute::Memory), + StorageKind::Postgres | StorageKind::MySql => { + let repo_id = self.storage.repo_id?; + let dsns = if self.storage.dsns.is_empty() { + vec![self.storage.dsn.clone()?] + } else { + self.storage.dsns.clone() + }; + Some(StorageRoute::Sharded { + dsns, + repo_id: Some(repo_id), + root: Some(root.to_string()), + }) + } + StorageKind::Sqlite | StorageKind::Lmdb | StorageKind::Redis => { + let dsn = self + .storage + .dsn + .clone() + .or_else(|| self.storage_dsn(root)); + Some(StorageRoute::Local(dsn?)) + } + } + } + + /// Sinh `repo_id` ngẫu nhiên (u64) nếu backend là RDBMS và config chưa có, + /// rồi ghi vào `[storage]` của `config.toml` (self-heal). Trả `Some(repo_id)` + /// nếu là RDBMS (kể cả khi đã có sẵn), `None` nếu không phải RDBMS. + pub fn ensure_repo_id(root: &Utf8Path) -> Option { + if !ExtractConfig::load(root).storage.kind.is_rdbms() { + return None; + } + if let Some(id) = ExtractConfig::load(root).storage.repo_id { + return Some(id); + } + let repo_id = { + let mut buf = [0u8; 8]; + let _ = getrandom::getrandom(&mut buf); + u64::from_le_bytes(buf) + }; + let path = root.join(".codegraph").join("config.toml"); + if let Ok(text) = fs::read_to_string(path.as_std_path()) { + let inserted = if let Some(idx) = text.find("[storage]") { + let header = "[storage]"; + let mut s = String::with_capacity(text.len() + 40); + s.push_str(&text[..idx]); + s.push_str(header); + s.push_str("\n# repo_id (partition key) — sinh bởi `codegraph init`.\n"); + s.push_str(&format!("repo_id = {repo_id}\n")); + s.push_str(&text[idx + header.len()..]); + s + } else { + format!("{text}\n[storage]\nrepo_id = {repo_id}\n") + }; + let _ = fs::write(path.as_std_path(), inserted); } + Some(repo_id) } } @@ -189,12 +283,15 @@ headers = "auto" # effect = "sql_query" [storage] -# Backend lưu index: "sqlite", "lmdb", "redis", hoặc "memory". +# Backend lưu index: "sqlite", "lmdb", "redis", "memory", "postgres", hoặc "mysql". type = "sqlite" # DSN override (mặc định dựng từ `type` + project path): # sqlite → sqlite:///.codegraph/db.sqlite # lmdb → lmdb:///.codegraph/db.lmdb # redis → bắt buộc khai dsn, ví dụ redis://localhost:6379 +# postgres/mysql → bắt buộc khai `dsns` (hoặc `dsn` nếu 1 shard), ví dụ: +# dsns = ["postgres://user:pass@db1:5432/codegraph", "postgres://user:pass@db2:5432/codegraph"] +# repo_id = 14028493579208694412 # sinh bởi `codegraph init` (partition key) # dsn = "sqlite:///tmp/codegraph.db" "#; diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 4947acf3b..4391a5dd9 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -41,11 +41,15 @@ pub use crate::storage::lmdb::LmdbStorage; #[cfg(feature = "sqlite")] pub use crate::storage::sqlite::SqliteStorage; pub use crate::storage::{InMemoryStorage, Storage, Tx}; +#[cfg(feature = "postgres")] +pub use crate::storage::postgres::PostgresStorage; +#[cfg(feature = "mysql")] +pub use crate::storage::mysql::MySqlStorage; use codegraph_core::{ CallRecord, CallSite, CallSiteResult, ClassInfo, DependenciesReport, Dependency, EdgeMeta, EffectType, Error, FileInfo, FlowCall, FlowResult, FunctionScope, MemberInfo, ResolveResult, - SYMBOL_BASE, SearchFlowResult, SemgraphStats, Symbol, SymbolKind, SymbolMatch, is_marker, - marker_name, + SYMBOL_BASE, SearchFlowResult, SemgraphStats, StorageRoute, Symbol, SymbolKind, SymbolMatch, + is_marker, marker_name, }; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -353,31 +357,106 @@ impl GraphIndex { Ok(idx) } - /// Mở index từ redis dsn (feature `redis`) — rebuild từ entity store. - #[cfg(feature = "postgres")] - async fn open_postgres_dispatch(path: &str) -> Result { - Self::open_postgres(path).await - } - #[cfg(feature = "postgres")] - async fn open_postgres(path: &str) -> Result { - let storage = crate::storage::postgres::PostgresStorage::open(path).await?; + /// Mở index trên Redis (`redis://` / `rediss://`). Keyspace prefix được + /// dẫn xuất từ số DB trong DSN (`redis://host:port/15` → `codegraph:idx:15`) + /// để nhiều index không đụng nhau. Redis không multi-tenant theo `repo_id` + /// như RDBMS — mỗi DB number = một index riêng. + #[cfg(feature = "redis")] + async fn open_redis(dsn: &str) -> Result { + let db = dsn + .trim_start_matches("rediss://") + .trim_start_matches("redis://") + .rsplit('/') + .next() + .and_then(|s| if s.is_empty() { None } else { s.parse::().ok() }) + .unwrap_or(0); + let client = redis::Client::open(dsn) + .map_err(|e| Error::Db(format!("redis client: {e}")))?; + let storage = crate::storage::redis::RedisStorage::new( + client, + &format!("codegraph:idx:{db}"), + ) + .await + .map_err(serr)?; let storage = Arc::new(RwLock::new(storage)) as Arc>; let mut idx = Self::new_with_storage(storage); idx.rebuild().await?; Ok(idx) } - #[cfg(feature = "mysql")] - async fn open_mysql_dispatch(path: &str) -> Result { - Self::open_mysql(path).await - } - #[cfg(feature = "mysql")] - async fn open_mysql(path: &str) -> Result { - let storage = crate::storage::mysql::MySqlStorage::open(path).await?; - let storage = Arc::new(RwLock::new(storage)) as Arc>; - let mut idx = Self::new_with_storage(storage); - idx.rebuild().await?; - Ok(idx) + /// Mở index theo `StorageRoute` — hỗ trợ multi-tenant + sharding RDBMS. + /// + /// - `Memory` → in-memory. + /// - `Local(dsn)` → `open(dsn)` (sqlite/lmdb/redis). + /// - `Sharded { dsns, repo_id, root }` → tính `shard = repo_id % N` + /// (`StorageRoute::shard_of`), mở backend per-repo (`PostgresStorage`/ + /// `MySqlStorage`) trên `dsns[shard]`, ensure row `repos`, rebuild. + pub async fn open_route(route: &StorageRoute) -> Result { + match route { + StorageRoute::Memory => Ok(Self::in_memory()), + StorageRoute::Local(dsn) => Self::open(dsn).await, + StorageRoute::Sharded { dsns, repo_id, .. } => { + let repo_id = repo_id.ok_or_else(|| { + Error::Db( + "StorageRoute::Sharded thiếu repo_id — chạy `codegraph init` để sinh" + .into(), + ) + })?; + if dsns.is_empty() { + return Err(Error::Db("StorageRoute::Sharded không có DSN nào".into())); + } + let shard = route.shard_of(repo_id).ok_or_else(|| { + Error::Db("không tính được shard từ StorageRoute::Sharded".into()) + })?; + let dsn = dsns.get(shard).ok_or_else(|| { + Error::Db(format!("shard {shard} vượt quá số lượng DSN")) + })?; + let result: Result = if dsn.starts_with("postgres://") { + #[cfg(feature = "postgres")] + { + let storage = crate::storage::postgres::PostgresStorage::open(dsn, repo_id) + .await + .map_err(serr)?; + storage + .ensure_registered(shard, route.root()) + .await + .map_err(serr)?; + let storage = Arc::new(RwLock::new(storage)) + as Arc>; + let mut idx = Self::new_with_storage(storage); + idx.rebuild().await?; + Ok(idx) + } + #[cfg(not(feature = "postgres"))] + { + Err(Error::Db("feature 'postgres' chưa bật".into())) + } + } else if dsn.starts_with("mysql://") { + #[cfg(feature = "mysql")] + { + let storage = crate::storage::mysql::MySqlStorage::open(dsn, repo_id) + .await + .map_err(serr)?; + storage + .ensure_registered(shard, route.root()) + .await + .map_err(serr)?; + let storage = Arc::new(RwLock::new(storage)) + as Arc>; + let mut idx = Self::new_with_storage(storage); + idx.rebuild().await?; + Ok(idx) + } + #[cfg(not(feature = "mysql"))] + { + Err(Error::Db("feature 'mysql' chưa bật".into())) + } + } else { + Err(Error::Db(format!("Sharded DSN scheme không hỗ trợ: {dsn}"))) + }; + result + } + } } diff --git a/crates/codegraph-graph/src/shared.rs b/crates/codegraph-graph/src/shared.rs index 6c328f08f..cd3358f81 100644 --- a/crates/codegraph-graph/src/shared.rs +++ b/crates/codegraph-graph/src/shared.rs @@ -1,20 +1,21 @@ //! SharedGraphIndex — index dùng chung cho production (GraphApi/MCP/viz). //! //! Mọi request dùng chung 1 snapshot `Arc`. Index sống trong một -//! backend persistent mà DSN chỉ rõ (`sqlite://...` / `lmdb://...` / `redis://...`): +//! backend persistent mà `StorageRoute` chỉ rõ (`sqlite://...` / `lmdb://...` / +//! `redis://...` / `Sharded{dsns, repo_id, ...}` cho Postgres/MySQL): //! `GraphIndex::ingest` (CLI/watcher, tiến trình riêng) bump `index_version` //! trong store; `ensure_fresh` probe version (đọc thẳng store — không cần //! sidecar) và rebuild snapshot khi stale dưới `rebuild_lock` (N request stale -//! đồng thời chỉ 1 lần rebuild), đổi snapshot dưới `RwLock`. `dsn = None`: +//! đồng thời chỉ 1 lần rebuild), đổi snapshot dưới `RwLock`. `route = None`: //! in-memory — không có writer ngoài, snapshot coi như luôn fresh sau lần //! build đầu. //! -//! DSN là **source duy nhất** cho cả `rebuild` (mở backend) lẫn `current_version` -//! (probe) — nên khi nhiều backend cùng được bật (vd `sqlite` + `lmdb`), backend -//! được chọn theo scheme trong DSN, không phải theo thứ tự feature. +//! `StorageRoute` là **source duy nhất** cho cả `rebuild` (mở backend) lẫn +//! `current_version` (probe) — nên khi nhiều backend cùng được bật, backend +//! được chọn theo scheme trong route, không phải theo thứ tự feature. use crate::GraphIndex; -use codegraph_core::Result; +use codegraph_core::{Result, StorageRoute}; use std::sync::Arc; use tokio::sync::{Mutex, RwLock}; @@ -32,25 +33,25 @@ struct IndexState { /// chiếu 1 instance. Rebuild đồng bộ theo version file — request đầu sau khi /// re-index xong chờ rebuild, các request sau thấy đã fresh. pub struct SharedGraphIndex { - /// DSN nơi persist index (`None` = in-memory, không có writer ngoài). - dsn: Option, + /// Route persist index (`None` = in-memory, không có writer ngoài). + route: Option, state: RwLock, /// Serialize rebuild — N request stale đồng thời chỉ 1 lần rebuild. rebuild_lock: Arc>, } impl SharedGraphIndex { - /// Mở index dùng chung. - /// - /// `dsn = Some(d)`: chưa build — `ensure_fresh` sẽ mở đúng backend theo - /// scheme rồi rebuild index từ store lần đầu. `dsn = None`: in-memory. - /// - /// `dsn` phải là DSN đầy đủ scheme (vd `sqlite:///path/db.sqlite`, - /// `lmdb:///path/db`) — không phải plain path, để nhiều backend cùng bật - /// vẫn chọn đúng backend. + /// Mở index dùng chung từ một DSN string (sqlite/lmdb/redis). Tiện ích bọc + /// `open_route` với `StorageRoute::Local`. pub async fn open(dsn: Option) -> Result { + Self::open_route(dsn.map(StorageRoute::Local)).await + } + + /// Mở index dùng chung theo `StorageRoute` (hỗ trợ multi-tenant + sharding + /// RDBMS). `route = None` → in-memory. + pub async fn open_route(route: Option) -> Result { Ok(Self { - dsn, + route, state: RwLock::new(IndexState { index: Arc::new(GraphIndex::in_memory()), version: 0, @@ -60,42 +61,88 @@ impl SharedGraphIndex { }) } - /// Scheme của DSN (`"sqlite"`, `"lmdb"`, `"redis"`) — `None` nếu in-memory. - fn scheme(&self) -> Option<&'static str> { - let dsn = self.dsn.as_ref()?; - if dsn.starts_with("sqlite://") { - return Some("sqlite"); - } - if dsn.starts_with("lmdb://") { - return Some("lmdb"); + /// Scheme của backend (`"sqlite"`, `"lmdb"`, `"redis"`, `"postgres"`, + /// `"mysql"`) — `None` nếu in-memory hoặc không đo được version độc lập. + fn backend_scheme(&self) -> Option<&'static str> { + let route = self.route.as_ref()?; + match route { + StorageRoute::Memory => None, + StorageRoute::Local(dsn) => { + if dsn.starts_with("sqlite://") { + Some("sqlite") + } else if dsn.starts_with("lmdb://") { + Some("lmdb") + } else if dsn.starts_with("redis://") { + Some("redis") + } else { + None + } + } + StorageRoute::Sharded { dsns, .. } => { + let dsn = dsns.first()?; + if dsn.starts_with("postgres://") { + Some("postgres") + } else if dsn.starts_with("mysql://") { + Some("mysql") + } else { + None + } + } } - if dsn.starts_with("redis://") { - return Some("redis"); + } + + /// Với route `Sharded`, giải shard → `(dsn, repo_id)` để probe/open. + #[cfg(any(feature = "postgres", feature = "mysql"))] + fn sharded_target(&self) -> Option<(String, u64)> { + let route = self.route.as_ref()?; + match route { + StorageRoute::Sharded { dsns, repo_id, .. } => { + let repo_id = (*repo_id)?; + let shard = route.shard_of(repo_id)?; + let dsn = dsns.get(shard)?; + Some((dsn.clone(), repo_id)) + } + _ => None, } - // Các scheme/DSN khác (chưa biết) — không đo được version độc lập. - None } /// Version index trên đĩa hiện tại — `None` nếu probe thất bại (store chưa - /// có hoặc đang bị re-index), hay backend không probe độc lập được (redis). - /// Chỉ gọi khi `dsn.is_some()`. + /// có hoặc đang bị re-index), hay backend không probe độc lập được (redis/ + /// in-memory/unknown scheme). async fn current_version(&self) -> Option { - let dsn = self.dsn.as_ref()?; - // `path` chỉ dùng bởi các backend có probe file độc lập (sqlite/lmdb); - // build không bật backend nào → biến thừa, cho phép bỏ qua lint. - #[cfg_attr( - not(any(feature = "sqlite", feature = "lmdb")), - allow(unused_variables) - )] - let path = trim_scheme(dsn); - match self.scheme() { + match self.backend_scheme()? { #[cfg(feature = "sqlite")] - Some("sqlite") => crate::storage::sqlite::SqliteStorage::probe_version(path) - .await - .ok(), + "sqlite" => { + let dsn = match &self.route { + Some(StorageRoute::Local(d)) => d.as_str(), + _ => return None, + }; + crate::storage::sqlite::SqliteStorage::probe_version(trim_scheme(dsn)) + .await + .ok() + } #[cfg(feature = "lmdb")] - Some("lmdb") => crate::storage::lmdb::probe_version(path).await.ok(), - // redis không có probe file ngoài — không đo được → stale. + "lmdb" => { + let dsn = match &self.route { + Some(StorageRoute::Local(d)) => d.as_str(), + _ => return None, + }; + crate::storage::lmdb::probe_version(trim_scheme(dsn)).await.ok() + } + #[cfg(feature = "postgres")] + "postgres" => { + let (dsn, repo_id) = self.sharded_target()?; + crate::storage::postgres::PostgresStorage::probe_version(&dsn, repo_id) + .await + .ok() + } + #[cfg(feature = "mysql")] + "mysql" => { + let (dsn, repo_id) = self.sharded_target()?; + crate::storage::mysql::MySqlStorage::probe_version(&dsn, repo_id) + .await + .ok() + } _ => None, } } @@ -104,7 +151,7 @@ impl SharedGraphIndex { /// → không có writer ngoài → luôn fresh. Backend không probe được (redis/ /// unknown scheme) → coi là stale để rebuilt lại. async fn is_fresh(&self, version: u64) -> bool { - if self.dsn.is_none() { + if self.route.is_none() { return true; } matches!(self.current_version().await, Some(v) if v == version) @@ -138,17 +185,13 @@ impl SharedGraphIndex { self.state.read().await.index.clone() } - /// Build index từ DSN hiện tại rồi swap snapshot (gọi trong `rebuild_lock`). - /// `GraphIndex::open` tự route theo scheme — không cần nhánh cfg. + /// Build index từ route hiện tại rồi swap snapshot (gọi trong `rebuild_lock`). + /// `GraphIndex::open_route` tự route theo scheme — không cần nhánh cfg. async fn rebuild_inner(&self) -> Result<()> { - #[cfg(any(feature = "sqlite", feature = "lmdb", feature = "redis"))] - let index = match &self.dsn { - Some(d) => GraphIndex::open(d).await?, + let index = match &self.route { + Some(route) => GraphIndex::open_route(route).await?, None => GraphIndex::in_memory(), }; - #[cfg(not(any(feature = "sqlite", feature = "lmdb", feature = "redis")))] - let index = GraphIndex::in_memory(); - let version = index.version(); let mut state = self.state.write().await; state.index = Arc::new(index); @@ -159,6 +202,7 @@ impl SharedGraphIndex { } /// Bỏ `scheme://` khỏi DSN — trả phần còn lại (path cho probe file). +#[cfg(any(feature = "sqlite", feature = "lmdb"))] fn trim_scheme(dsn: &str) -> &str { dsn.strip_prefix("sqlite://") .or_else(|| dsn.strip_prefix("lmdb://")) diff --git a/crates/codegraph-graph/src/storage/mysql.rs b/crates/codegraph-graph/src/storage/mysql.rs index f2ddb4735..21074cb3d 100644 --- a/crates/codegraph-graph/src/storage/mysql.rs +++ b/crates/codegraph-graph/src/storage/mysql.rs @@ -1,40 +1,95 @@ use async_trait::async_trait; -use sqlx::{mysql::MySqlPoolOptions, MySqlPool}; -use super::{Result, Storage, StorageError, Tx, EMPTY}; +use sqlx::mysql::{MySqlPoolOptions, MySqlRow}; +use sqlx::{MySqlPool, Row}; +use super::{decode_chain, encode_chain, Result, Storage, StorageError, Tx}; +use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; -/// MySQL implementation of the `Storage` trait. -/// The schema mirrors the SQLite version, adjusted for MySQL syntax. +/// MySQL implementation của `Storage` trait — multi-tenant (mọi bảng dẫn đầu bằng +/// `repo_id`), theo thiết kế `sql/README.md`. Instance được bind vào một `repo_id`. +/// Schema apply **thủ công** (user chạy `sql/mysql/001`+`002`); code chỉ seed +/// runtime row per-repo. pub struct MySqlStorage { pool: MySqlPool, + repo_id: u64, +} + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn ser_err(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) } impl MySqlStorage { - /// Open a MySQL connection pool. `dsn` must be a valid MySQL URL - /// (e.g. `mysql://user:pass@host:3306/db`). No automatic initialization – - /// the schema should be applied manually (e.g. via the migration files). - pub async fn open(dsn: &str) -> Result { + /// Mở pool + seed per-repo runtime rows. `repo_id` do config/sharding quyết + /// định (không lấy từ DSN). + pub async fn open(dsn: &str, repo_id: u64) -> Result { let pool = MySqlPoolOptions::new() .max_connections(5) .connect(dsn) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - Ok(Self { pool }) + .map_err(db_err)?; + let s = Self { pool, repo_id }; + s.ensure_repo_seeded().await?; + Ok(s) } - // MySQL returns a `u64` for `LAST_INSERT_ID`, but our node ids live in a - // central `rt_counter` table shared by all shards. Mirror the sequence used - // by the other backends: read the current `next`, then bump it. - async fn reserve_node_id(&self) -> Result { - let row: (i64,) = sqlx::query_as("SELECT next FROM rt_counter WHERE id = 1") - .fetch_one(&self.pool) + async fn ensure_repo_seeded(&self) -> Result<()> { + let rid = self.repo_id as i64; + sqlx::query( + "INSERT IGNORE INTO rt_nodes (repo_id, id, prefix, record) VALUES (?, 0, '', 0)", + ) + .bind(rid) + .execute(&self.pool) + .await + .map_err(db_err)?; + sqlx::query("INSERT IGNORE INTO rt_counter (repo_id, next) VALUES (?, 1)") + .bind(rid) + .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - let id = row.0 as usize; - sqlx::query("UPDATE rt_counter SET next = next + 1 WHERE id = 1") + .map_err(db_err)?; + sqlx::query("INSERT IGNORE INTO sg_next_id (repo_id, next) VALUES (?, 100)") + .bind(rid) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - Ok(id) + .map_err(db_err)?; + sqlx::query("INSERT IGNORE INTO sg_meta (repo_id, version) VALUES (?, 0)") + .bind(rid) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + /// Ghi/ensure row `repos` (registry toàn cục, shard của repo) — idempotent. + /// Bảng này nằm trong migration `002` (áp dụng thủ công). + pub async fn ensure_registered(&self, shard: usize, root: Option<&str>) -> Result<()> { + sqlx::query("INSERT IGNORE INTO repos (repo_id, shard, root) VALUES (?, ?, ?)") + .bind(self.repo_id as i64) + .bind(shard as i32) + .bind(root) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + /// Cấp node id nguyên tử, per-repo — dùng idiom `LAST_INSERT_ID(next) + 1` + /// (theo README) trong 1 transaction để tránh race. + async fn reserve_node_id(&self) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("UPDATE rt_counter SET next = LAST_INSERT_ID(next) + 1 WHERE repo_id = ?") + .bind(self.repo_id as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + let row: (u64,) = sqlx::query_as("SELECT LAST_INSERT_ID()") + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(row.0 as usize) } } @@ -42,13 +97,17 @@ impl MySqlStorage { impl Storage for MySqlStorage { async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { let id = self.reserve_node_id().await?; - sqlx::query("INSERT INTO rt_nodes (id, prefix, record) VALUES (?, ?, ?)") - .bind(id as i64) - .bind(prefix) - .bind(record as i64) - .execute(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + sqlx::query( + "INSERT INTO rt_nodes (repo_id, id, prefix, record) VALUES (?, ?, ?, ?) \ + ON DUPLICATE KEY UPDATE id = id", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .bind(prefix) + .bind(record as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; Ok(id) } @@ -58,33 +117,37 @@ impl Storage for MySqlStorage { prefix: Option>, record: Option, ) -> Result<()> { + let rid = self.repo_id as i64; if let Some(p) = prefix { - sqlx::query("UPDATE rt_nodes SET prefix = ? WHERE id = ?") + sqlx::query("UPDATE rt_nodes SET prefix = ? WHERE repo_id = ? AND id = ?") .bind(p) + .bind(rid) .bind(id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; } if let Some(r) = record { - sqlx::query("UPDATE rt_nodes SET record = ? WHERE id = ?") + sqlx::query("UPDATE rt_nodes SET record = ? WHERE repo_id = ? AND id = ?") .bind(r as i64) + .bind(rid) .bind(id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; } Ok(()) } async fn get_node(&self, id: usize) -> Result<(Vec, usize)> { let row = sqlx::query_as::<_, (Vec, i64)>( - "SELECT prefix, record FROM rt_nodes WHERE id = ?", + "SELECT prefix, record FROM rt_nodes WHERE repo_id = ? AND id = ?", ) + .bind(self.repo_id as i64) .bind(id as i64) .fetch_optional(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; let Some((prefix, record)) = row else { return Err(StorageError::BranchOutOfRange(id)); }; @@ -93,34 +156,39 @@ impl Storage for MySqlStorage { async fn get_children(&self, id: usize) -> Result> { let rows = sqlx::query_as::<_, (i64,)>( - "SELECT child FROM rt_children WHERE parent = ? ORDER BY child", + "SELECT child FROM rt_children WHERE repo_id = ? AND parent = ? ORDER BY child", ) + .bind(self.repo_id as i64) .bind(id as i64) .fetch_all(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(rows.into_iter().map(|(c,)| c as usize).collect()) } async fn set_root(&mut self, shard: usize, root: usize) -> Result<()> { sqlx::query( - "INSERT INTO rt_roots (shard, root) VALUES (?, ?) \ + "INSERT INTO rt_roots (repo_id, shard, root) VALUES (?, ?, ?) \ ON DUPLICATE KEY UPDATE root = VALUES(root)", ) - .bind(shard as i64) + .bind(self.repo_id as i64) + .bind(shard as i32) .bind(root as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_root(&self, shard: usize) -> Result { - let row = sqlx::query_as::<_, (i64,)>("SELECT root FROM rt_roots WHERE shard = ?") - .bind(shard as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (i64,)>( + "SELECT root FROM rt_roots WHERE repo_id = ? AND shard = ?", + ) + .bind(self.repo_id as i64) + .bind(shard as i32) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; let Some((root,)) = row else { return Err(StorageError::BranchOutOfRange(shard)); }; @@ -129,119 +197,736 @@ impl Storage for MySqlStorage { async fn set_meta(&mut self, record: usize, meta: &[u8]) -> Result<()> { sqlx::query( - "INSERT INTO rt_meta (record, meta) VALUES (?, ?) \ + "INSERT INTO rt_meta (repo_id, record, meta) VALUES (?, ?, ?) \ ON DUPLICATE KEY UPDATE meta = VALUES(meta)", ) + .bind(self.repo_id as i64) .bind(record as i64) .bind(meta) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_meta(&self, record: usize) -> Result>> { - let row = sqlx::query_as::<_, (Vec,)>("SELECT meta FROM rt_meta WHERE record = ?") - .bind(record as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT meta FROM rt_meta WHERE repo_id = ? AND record = ?", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; Ok(row.map(|(m,)| m)) } async fn set_key_len(&mut self, record: usize, len: usize) -> Result<()> { sqlx::query( - "INSERT INTO rt_keylen (record, len) VALUES (?, ?) \ + "INSERT INTO rt_keylen (repo_id, record, len) VALUES (?, ?, ?) \ ON DUPLICATE KEY UPDATE len = VALUES(len)", ) + .bind(self.repo_id as i64) .bind(record as i64) - .bind(len as i64) + .bind(len as i32) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_key_len(&self, record: usize) -> Result> { - let row = sqlx::query_as::<_, (i64,)>("SELECT len FROM rt_keylen WHERE record = ?") - .bind(record as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (i32,)>( + "SELECT len FROM rt_keylen WHERE repo_id = ? AND record = ?", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; Ok(row.map(|(len,)| len as usize)) } async fn add_shortcut_node(&mut self, shard: usize, elem: &[u8], node_id: usize) -> Result<()> { sqlx::query( - "INSERT IGNORE INTO rt_shortcuts (shard, elem, node_id) VALUES (?, ?, ?)", + "INSERT IGNORE INTO rt_shortcuts (repo_id, shard, elem, node_id) VALUES (?, ?, ?, ?)", ) - .bind(shard as i64) + .bind(self.repo_id as i64) + .bind(shard as i32) .bind(elem) .bind(node_id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_shortcut_nodes(&self, shard: usize, elem: &[u8]) -> Result> { let rows = sqlx::query_as::<_, (i64,)>( - "SELECT node_id FROM rt_shortcuts WHERE shard = ? AND elem = ?", + "SELECT node_id FROM rt_shortcuts WHERE repo_id = ? AND shard = ? AND elem = ?", ) - .bind(shard as i64) + .bind(self.repo_id as i64) + .bind(shard as i32) .bind(elem) .fetch_all(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(rows.into_iter().map(|(id,)| id as usize).collect()) } async fn clear_shortcuts(&mut self) -> Result<()> { - sqlx::query("DELETE FROM rt_shortcuts") + sqlx::query("DELETE FROM rt_shortcuts WHERE repo_id = ?") + .bind(self.repo_id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn set_edge_data(&mut self, edge: usize, data: &[u8]) -> Result<()> { sqlx::query( - "INSERT INTO rt_edges (id, data) VALUES (?, ?) \ + "INSERT INTO rt_edges (repo_id, id, data) VALUES (?, ?, ?) \ ON DUPLICATE KEY UPDATE data = VALUES(data)", ) + .bind(self.repo_id as i64) .bind(edge as i64) .bind(data) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_edge_data(&self, edge: usize) -> Result>> { - let row = sqlx::query_as::<_, (Vec,)>("SELECT data FROM rt_edges WHERE id = ?") - .bind(edge as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT data FROM rt_edges WHERE repo_id = ? AND id = ?", + ) + .bind(self.repo_id as i64) + .bind(edge as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; Ok(row.map(|(d,)| d)) } async fn clear_edges(&mut self) -> Result<()> { - sqlx::query("DELETE FROM rt_edges") + sqlx::query("DELETE FROM rt_edges WHERE repo_id = ?") + .bind(self.repo_id as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn for_each_edge_data( + &self, + f: &mut (dyn for<'a> FnMut(usize, &'a [u8]) -> Result<()> + Send), + ) -> Result<()> { + let rows = sqlx::query("SELECT id, data FROM rt_edges WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + for r in &rows { + let id: i64 = r.try_get("id").map_err(db_err)?; + let data: Vec = r.try_get("data").map_err(db_err)?; + f(id as usize, &data)?; + } + Ok(()) + } + + async fn set_node_meta(&mut self, elem: usize, meta: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_node_meta (repo_id, elem, meta) VALUES (?, ?, ?) \ + ON DUPLICATE KEY UPDATE meta = VALUES(meta)", + ) + .bind(self.repo_id as i64) + .bind(elem as i64) + .bind(meta) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn get_node_meta(&self, elem: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT meta FROM rt_node_meta WHERE repo_id = ? AND elem = ?", + ) + .bind(self.repo_id as i64) + .bind(elem as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(m,)| m)) + } + + async fn clear_node_meta(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_node_meta WHERE repo_id = ?") + .bind(self.repo_id as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn set_chain(&mut self, record: usize, chain: &[u64]) -> Result<()> { + let bytes = encode_chain(chain); + sqlx::query( + "INSERT INTO rt_chains (repo_id, record, chain) VALUES (?, ?, ?) \ + ON DUPLICATE KEY UPDATE chain = VALUES(chain)", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .bind(bytes) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn get_chain(&self, record: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT chain FROM rt_chains WHERE repo_id = ? AND record = ?", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| decode_chain(&b))) + } + + async fn clear_chains(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_chains WHERE repo_id = ?") + .bind(self.repo_id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; + Ok(()) + } + + async fn save_symbol(&mut self, sym: &Symbol) -> Result<()> { + let annotations = serde_json::to_string(&sym.annotations).map_err(ser_err)?; + sqlx::query( + "INSERT INTO sg_symbols \ + (repo_id, id, name, kind, scope, scope_id, type_ref, type_name, file, \ + line, end_line, signature, doc, annotations, language) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON DUPLICATE KEY UPDATE \ + name = VALUES(name), kind = VALUES(kind), scope = VALUES(scope), \ + scope_id = VALUES(scope_id), type_ref = VALUES(type_ref), \ + type_name = VALUES(type_name), file = VALUES(file), line = VALUES(line), \ + end_line = VALUES(end_line), signature = VALUES(signature), doc = VALUES(doc), \ + annotations = VALUES(annotations), language = VALUES(language)", + ) + .bind(self.repo_id as i64) + .bind(sym.id as i64) + .bind(&sym.name) + .bind(sym.kind.as_str()) + .bind(sym.scope.as_str()) + .bind(sym.scope_id as i64) + .bind(sym.type_ref as i64) + .bind(&sym.type_name) + .bind(&sym.file) + .bind(sym.line as i32) + .bind(sym.end_line as i32) + .bind(&sym.signature) + .bind(&sym.doc) + .bind(annotations) + .bind(&sym.language) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_symbol(&self, id: u64) -> Result> { + let row = sqlx::query( + "SELECT id, name, kind, scope, scope_id, type_ref, type_name, file, line, \ + end_line, signature, doc, annotations, language \ + FROM sg_symbols WHERE repo_id = ? AND id = ?", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.as_ref().map(row_to_symbol).transpose()?) + } + + async fn load_all_symbols(&self) -> Result> { + let rows = sqlx::query( + "SELECT id, name, kind, scope, scope_id, type_ref, type_name, file, line, \ + end_line, signature, doc, annotations, language FROM sg_symbols WHERE repo_id = ?", + ) + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.iter().map(row_to_symbol).collect() + } + + async fn save_next_id(&mut self, next: u64) -> Result<()> { + sqlx::query( + "INSERT INTO sg_next_id (repo_id, next) VALUES (?, ?) \ + ON DUPLICATE KEY UPDATE next = VALUES(next)", + ) + .bind(self.repo_id as i64) + .bind(next as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_next_id(&self) -> Result { + let row: Option<(i64,)> = sqlx::query_as("SELECT next FROM sg_next_id WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(v,)| v as u64).unwrap_or(0)) + } + + async fn all_chains(&self) -> Result)>> { + let rows = sqlx::query("SELECT record, chain FROM rt_chains WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + let record: i64 = r.try_get("record").map_err(db_err)?; + let chain: Vec = r.try_get("chain").map_err(db_err)?; + out.push((record as u64, chain)); + } + Ok(out) + } + + async fn set_call_records(&mut self, func: u64, records: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO sg_call_records (repo_id, func, records) VALUES (?, ?, ?) \ + ON DUPLICATE KEY UPDATE records = VALUES(records)", + ) + .bind(self.repo_id as i64) + .bind(func as i64) + .bind(records) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn get_call_records(&self, func: u64) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT records FROM sg_call_records WHERE repo_id = ? AND func = ?", + ) + .bind(self.repo_id as i64) + .bind(func as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| b)) + } + + async fn all_call_records(&self) -> Result)>> { + let rows = sqlx::query("SELECT func, records FROM sg_call_records WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + let func: i64 = r.try_get("func").map_err(db_err)?; + let records: Vec = r.try_get("records").map_err(db_err)?; + out.push((func as u64, records)); + } + Ok(out) + } + + async fn set_call_name_index(&mut self, name: &str, sites: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO sg_call_names (repo_id, name, sites) VALUES (?, ?, ?) \ + ON DUPLICATE KEY UPDATE sites = VALUES(sites)", + ) + .bind(self.repo_id as i64) + .bind(name) + .bind(sites) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_call_name_index(&self, name: &str) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT sites FROM sg_call_names WHERE repo_id = ? AND name = ?", + ) + .bind(self.repo_id as i64) + .bind(name) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| b)) + } + + async fn all_call_name_indexes(&self) -> Result)>> { + let rows = sqlx::query("SELECT name, sites FROM sg_call_names WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + let name: String = r.try_get("name").map_err(db_err)?; + let sites: Vec = r.try_get("sites").map_err(db_err)?; + out.push((name, sites)); + } + Ok(out) + } + + async fn upsert_file(&mut self, f: &FileInfo) -> Result<()> { + sqlx::query( + "INSERT INTO sg_files (repo_id, path, language, bytes, lines) VALUES (?, ?, ?, ?, ?) \ + ON DUPLICATE KEY UPDATE \ + language = VALUES(language), bytes = VALUES(bytes), lines = VALUES(lines)", + ) + .bind(self.repo_id as i64) + .bind(&f.path) + .bind(&f.language) + .bind(f.bytes as i64) + .bind(f.lines as i32) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_all_files(&self) -> Result> { + let rows = sqlx::query("SELECT path, language, bytes, lines FROM sg_files WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + out.push(FileInfo { + path: r.try_get("path").map_err(db_err)?, + language: r.try_get("language").map_err(db_err)?, + bytes: r.try_get::("bytes").map_err(db_err)? as u64, + lines: r.try_get::("lines").map_err(db_err)? as u32, + }); + } + Ok(out) + } + + async fn version(&self) -> Result { + let row: Option<(i64,)> = sqlx::query_as("SELECT version FROM sg_meta WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(v,)| v as u64).unwrap_or(0)) + } + + async fn set_version(&mut self, v: u64) -> Result<()> { + sqlx::query( + "INSERT INTO sg_meta (repo_id, version) VALUES (?, ?) \ + ON DUPLICATE KEY UPDATE version = VALUES(version)", + ) + .bind(self.repo_id as i64) + .bind(v as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn clear_entities(&mut self) -> Result<()> { + let rid = self.repo_id as i64; + let mut tx = self.pool.begin().await.map_err(db_err)?; + for t in [ + "sg_symbols", + "sg_files", + "sg_call_records", + "sg_call_names", + "rt_nodes", + "rt_children", + "rt_roots", + "rt_meta", + "rt_keylen", + "rt_shortcuts", + "rt_chains", + "rt_edges", + "rt_node_meta", + "rt_node_blooms", + ] { + sqlx::query(&format!("DELETE FROM {t} WHERE repo_id = ?")) + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + sqlx::query("UPDATE rt_counter SET next = 1 WHERE repo_id = ?") + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query("UPDATE sg_next_id SET next = 100 WHERE repo_id = ?") + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query("UPDATE sg_meta SET version = 0 WHERE repo_id = ?") + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query("INSERT IGNORE INTO rt_nodes (repo_id, id, prefix, record) VALUES (?, 0, '', 0)") + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; Ok(()) } - // The remaining methods are either no‑ops or can be forwarded to other - // storage implementations if needed. For now we keep the minimal set. - async fn set_node_meta(&mut self, _elem: usize, _meta: &[u8]) -> Result<()> { Ok(()) } - async fn get_node_meta(&self, _elem: usize) -> Result>> { Ok(None) } - async fn clear_node_meta(&mut self) -> Result<()> { Ok(()) } - async fn set_chain(&mut self, _record: usize, _chain: &[u64]) -> Result<()> { Ok(()) } - async fn get_chain(&self, _record: usize) -> Result>> { Ok(None) } - async fn clear_chains(&mut self) -> Result<()> { Ok(()) } - async fn save_symbol(&mut self, _sym: &codegraph_core::Symbol) -> Result<()> { Ok(()) } - async fn load_symbol(&self, _id: u64) -> Result> { Ok(None) } + #[cfg(feature = "bloom-search")] + async fn set_node_bloom(&mut self, id: usize, bloom: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_node_blooms (repo_id, id, bloom) VALUES (?, ?, ?) \ + ON DUPLICATE KEY UPDATE bloom = VALUES(bloom)", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .bind(bloom) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + #[cfg(feature = "bloom-search")] + async fn get_node_bloom(&self, id: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT bloom FROM rt_node_blooms WHERE repo_id = ? AND id = ?", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| b)) + } + + fn new_tx(&self) -> Box { + Box::new(MySqlTx { + pool: self.pool.clone(), + repo_id: self.repo_id, + nodes: Vec::new(), + ops: Vec::new(), + }) + } +} + +/// Probe version index trên đĩa (dùng cho `SharedGraphIndex::ensure_fresh`). +#[cfg(feature = "mysql")] +impl MySqlStorage { + pub async fn probe_version(dsn: &str, repo_id: u64) -> Result { + let pool = MySqlPoolOptions::new() + .max_connections(2) + .connect(dsn) + .await + .map_err(db_err)?; + let row: Option<(i64,)> = sqlx::query_as("SELECT version FROM sg_meta WHERE repo_id = ?") + .bind(repo_id as i64) + .fetch_optional(&pool) + .await + .map_err(db_err)?; + Ok(row.map(|(v,)| v as u64).unwrap_or(0)) + } +} + +// ==================== MySqlTx ==================== + +/// Transaction cho `MySqlStorage`: buffer mutation, áp dụng atomic trong 1 MySQL +/// transaction tại `commit`. `new_node` cấp id nguyên tử per-repo (LAST_INSERT_ID). +pub struct MySqlTx { + pool: MySqlPool, + repo_id: u64, + nodes: Vec<(usize, Vec, usize)>, + ops: Vec, +} + +#[async_trait] +impl Tx for MySqlTx { + async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("UPDATE rt_counter SET next = LAST_INSERT_ID(next) + 1 WHERE repo_id = ?") + .bind(self.repo_id as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + let row: (u64,) = sqlx::query_as("SELECT LAST_INSERT_ID()") + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + let id = row.0 as usize; + self.nodes.push((id, prefix, record)); + Ok(id) + } + + async fn update_node( + &mut self, + id: usize, + prefix: Option>, + record: Option, + ) -> Result<()> { + self.ops.push(super::TxOp::UpdateNode { id, prefix, record }); + Ok(()) + } + + async fn add_child(&mut self, parent: usize, child: usize) -> Result<()> { + self.ops.push(super::TxOp::AddChild { parent, child }); + Ok(()) + } + + async fn move_child(&mut self, from: usize, to: usize, child: usize) -> Result<()> { + self.ops.push(super::TxOp::MoveChild { from, to, child }); + Ok(()) + } + + async fn commit(self: Box) -> Result<()> { + let MySqlTx { + pool, + repo_id, + nodes, + ops, + } = *self; + let rid = repo_id as i64; + let mut tx = pool.begin().await.map_err(db_err)?; + + for (id, prefix, record) in &nodes { + sqlx::query( + "INSERT INTO rt_nodes (repo_id, id, prefix, record) VALUES (?, ?, ?, ?) \ + ON DUPLICATE KEY UPDATE id = id", + ) + .bind(rid) + .bind(*id as i64) + .bind(prefix) + .bind(*record as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + + if let Some(max_id) = nodes.iter().map(|(id, _, _)| *id).max() { + sqlx::query("UPDATE rt_counter SET next = GREATEST(next, ?) WHERE repo_id = ?") + .bind((max_id + 1) as i64) + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + + for op in ops { + match op { + super::TxOp::AddChild { parent, child } => { + sqlx::query( + "INSERT IGNORE INTO rt_children (repo_id, parent, child) VALUES (?, ?, ?)", + ) + .bind(rid) + .bind(parent as i64) + .bind(child as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + super::TxOp::MoveChild { from, to, child } => { + sqlx::query( + "DELETE FROM rt_children WHERE repo_id = ? AND parent = ? AND child = ?", + ) + .bind(rid) + .bind(from as i64) + .bind(child as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query( + "INSERT IGNORE INTO rt_children (repo_id, parent, child) VALUES (?, ?, ?)", + ) + .bind(rid) + .bind(to as i64) + .bind(child as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + super::TxOp::UpdateNode { id, prefix, record } => { + if let Some(p) = prefix { + sqlx::query("UPDATE rt_nodes SET prefix = ? WHERE repo_id = ? AND id = ?") + .bind(p) + .bind(rid) + .bind(id as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + if let Some(r) = record { + sqlx::query("UPDATE rt_nodes SET record = ? WHERE repo_id = ? AND id = ?") + .bind(r as i64) + .bind(rid) + .bind(id as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + } + } + } + + tx.commit().await.map_err(db_err)?; + Ok(()) + } +} + +/// Map một row `sg_symbols` → `codegraph_core::Symbol`. +fn row_to_symbol(row: &MySqlRow) -> Result { + let id: i64 = row.try_get("id").map_err(db_err)?; + let name: String = row.try_get("name").map_err(db_err)?; + let kind: String = row.try_get("kind").map_err(db_err)?; + let scope: String = row.try_get("scope").map_err(db_err)?; + let scope_id: i64 = row.try_get("scope_id").map_err(db_err)?; + let type_ref: i64 = row.try_get("type_ref").map_err(db_err)?; + let type_name: Option = row.try_get("type_name").map_err(db_err)?; + let file: String = row.try_get("file").map_err(db_err)?; + let line: i32 = row.try_get("line").map_err(db_err)?; + let end_line: i32 = row.try_get("end_line").map_err(db_err)?; + let signature: Option = row.try_get("signature").map_err(db_err)?; + let doc: Option = row.try_get("doc").map_err(db_err)?; + let annotations: String = row.try_get("annotations").map_err(db_err)?; + let language: String = row.try_get("language").map_err(db_err)?; + let kind = SymbolKind::parse(&kind) + .ok_or_else(|| StorageError::Internal(format!("bad symbol kind: {kind}")))?; + let scope = ScopeLevel::parse(&scope) + .ok_or_else(|| StorageError::Internal(format!("bad scope level: {scope}")))?; + let annotations: Vec = serde_json::from_str(&annotations).map_err(ser_err)?; + Ok(Symbol { + id: id as u64, + name, + kind, + scope, + scope_id: scope_id as u64, + type_ref: type_ref as u64, + type_name, + file, + line: line as u32, + end_line: end_line as u32, + signature, + doc, + annotations, + language, + }) } diff --git a/crates/codegraph-graph/src/storage/postgres.rs b/crates/codegraph-graph/src/storage/postgres.rs index d0535fb0c..f6b0d74a9 100644 --- a/crates/codegraph-graph/src/storage/postgres.rs +++ b/crates/codegraph-graph/src/storage/postgres.rs @@ -1,110 +1,163 @@ use async_trait::async_trait; -use sqlx::{postgres::PgPoolOptions, PgPool}; -use super::{Result, Storage, StorageError, Tx, EMPTY}; +use sqlx::postgres::{PgPoolOptions, PgRow}; +use sqlx::{PgPool, Row}; +use super::{decode_chain, encode_chain, Result, Storage, StorageError, Tx}; +use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; -/// PostgreSQL implementation of the `Storage` trait. -/// The schema mirrors the SQLite version, adjusted for PostgreSQL syntax. +/// PostgreSQL implementation của `Storage` trait — multi-tenant theo thiết kế +/// `sql/README.md`: mọi bảng dẫn đầu bằng `repo_id`, 1 repository = 1 partition. +/// +/// Instance này được **bind vào một `repo_id`** (không đổi signature `Storage`, +/// không động tới backend khác). Schema được apply **thủ công** (user chạy +/// `sql/postgres/001`+`002`); code chỉ seed các runtime row per-repo (counter, +/// sentinel node, version). pub struct PostgresStorage { pool: PgPool, + repo_id: u64, +} + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn ser_err(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) } impl PostgresStorage { - /// Open a PostgreSQL connection pool. `dsn` must be a valid Postgres URL - /// (e.g. `postgres://user:pass@host:5432/db`). No automatic initialization – - /// the schema should be applied manually (e.g. via the migration files). - pub async fn open(dsn: &str) -> Result { + /// Mở pool + seed per-repo runtime rows. `repo_id` do config/sharding quyết + /// định (không lấy từ DSN). `dsn` phải là Postgres URL hợp lệ. + pub async fn open(dsn: &str, repo_id: u64) -> Result { let pool = PgPoolOptions::new() .max_connections(5) .connect(dsn) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - Ok(Self { pool }) - } - - - async fn init(&mut self) -> Result<()> { - // Same tables as SQLite, using PostgreSQL types. - for stmt in [ - "CREATE TABLE IF NOT EXISTS rt_nodes (\n id BIGSERIAL PRIMARY KEY,\n prefix BYTEA NOT NULL,\n record BIGINT NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS rt_children (\n parent BIGINT NOT NULL,\n child BIGINT NOT NULL,\n PRIMARY KEY (parent, child)\n )", - "CREATE INDEX IF NOT EXISTS idx_rt_children_parent ON rt_children(parent)", - "CREATE TABLE IF NOT EXISTS rt_roots (\n shard BIGINT PRIMARY KEY,\n root BIGINT NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS rt_meta (\n record BIGINT PRIMARY KEY,\n meta BYTEA NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS rt_keylen (\n record BIGINT PRIMARY KEY,\n len BIGINT NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS rt_shortcuts (\n shard BIGINT NOT NULL,\n elem BYTEA NOT NULL,\n node_id BIGINT NOT NULL,\n PRIMARY KEY (shard, elem, node_id)\n )", - "CREATE INDEX IF NOT EXISTS idx_rt_shortcuts_lookup ON rt_shortcuts(shard, elem)", - "CREATE TABLE IF NOT EXISTS rt_edges (\n id BIGINT PRIMARY KEY,\n data BYTEA NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS rt_node_meta (\n elem BIGINT PRIMARY KEY,\n meta BYTEA NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS rt_chains (\n record BIGINT PRIMARY KEY,\n chain BYTEA NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS rt_counter (\n id BIGINT PRIMARY KEY CHECK (id = 1),\n next BIGINT NOT NULL\n )", - // Entity tables needed for the rest of the graph. - "CREATE TABLE IF NOT EXISTS sg_symbols (\n id BIGINT PRIMARY KEY,\n data BYTEA NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS sg_next_id (\n id BIGINT PRIMARY KEY CHECK (id = 1),\n next BIGINT NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS sg_call_records (\n func BIGINT PRIMARY KEY,\n records BYTEA NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS sg_call_names (\n name TEXT PRIMARY KEY,\n sites BYTEA NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS sg_files (\n path TEXT PRIMARY KEY,\n language TEXT NOT NULL,\n bytes BIGINT NOT NULL,\n lines BIGINT NOT NULL\n )", - "CREATE TABLE IF NOT EXISTS sg_meta (\n id BIGINT PRIMARY KEY CHECK (id = 1),\n version BIGINT NOT NULL\n )", - // Initialise counters if they do not exist. - "INSERT INTO rt_counter (id, next) VALUES (1, 1) ON CONFLICT (id) DO NOTHING", - "INSERT INTO sg_next_id (id, next) VALUES (1, 100) ON CONFLICT (id) DO NOTHING", - "INSERT INTO sg_meta (id, version) VALUES (1, 0) ON CONFLICT (id) DO NOTHING", - ].iter() { - sqlx::query(stmt) - .execute(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } + .map_err(db_err)?; + let s = Self { pool, repo_id }; + s.ensure_repo_seeded().await?; + Ok(s) + } + + /// Idempotent seed runtime row cho 1 repo (theo mẫu "[Seed per repo]" trong + /// `sql/postgres/001`). KHÔNG tạo schema — schema là manual migration. + async fn ensure_repo_seeded(&self) -> Result<()> { + let rid = self.repo_id as i64; + sqlx::query( + "INSERT INTO rt_nodes (repo_id, id, prefix, record) VALUES ($1, 0, '', 0) \ + ON CONFLICT (repo_id, id) DO NOTHING", + ) + .bind(rid) + .execute(&self.pool) + .await + .map_err(db_err)?; + sqlx::query( + "INSERT INTO rt_counter (repo_id, next) VALUES ($1, 1) ON CONFLICT (repo_id) DO NOTHING", + ) + .bind(rid) + .execute(&self.pool) + .await + .map_err(db_err)?; + sqlx::query( + "INSERT INTO sg_next_id (repo_id, next) VALUES ($1, 100) ON CONFLICT (repo_id) DO NOTHING", + ) + .bind(rid) + .execute(&self.pool) + .await + .map_err(db_err)?; + sqlx::query( + "INSERT INTO sg_meta (repo_id, version) VALUES ($1, 0) ON CONFLICT (repo_id) DO NOTHING", + ) + .bind(rid) + .execute(&self.pool) + .await + .map_err(db_err)?; Ok(()) } + + /// Ghi/ensure row `repos` (registry toàn cục, shard của repo) — idempotent. + /// Bảng này nằm trong migration `002` (áp dụng thủ công). + pub async fn ensure_registered(&self, shard: usize, root: Option<&str>) -> Result<()> { + sqlx::query( + "INSERT INTO repos (repo_id, shard, root) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id) DO NOTHING", + ) + .bind(self.repo_id as i64) + .bind(shard as i32) + .bind(root) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + /// Cấp node id nguyên tử, per-repo. + async fn reserve_node_id(&self) -> Result { + let row: (i64,) = sqlx::query_as( + "UPDATE rt_counter SET next = next + 1 WHERE repo_id = $1 RETURNING next - 1", + ) + .bind(self.repo_id as i64) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + Ok(row.0 as usize) + } } #[async_trait] impl Storage for PostgresStorage { async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { - // Reserve an id via the counter table. - let row: (i64,) = sqlx::query_as( - "UPDATE rt_counter SET next = next + 1 WHERE id = 1 RETURNING next - 1", + let id = self.reserve_node_id().await?; + sqlx::query( + "INSERT INTO rt_nodes (repo_id, id, prefix, record) VALUES ($1, $2, $3, $4) \ + ON CONFLICT (repo_id, id) DO NOTHING", ) - .fetch_one(&self.pool) + .bind(self.repo_id as i64) + .bind(id as i64) + .bind(prefix) + .bind(record as i64) + .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - let id = row.0 as usize; - sqlx::query("INSERT INTO rt_nodes (id, prefix, record) VALUES ($1, $2, $3)") - .bind(id as i64) - .bind(prefix) - .bind(record as i64) - .execute(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(id) } - async fn update_node(&mut self, id: usize, prefix: Option>, record: Option) -> Result<()> { + async fn update_node( + &mut self, + id: usize, + prefix: Option>, + record: Option, + ) -> Result<()> { + let rid = self.repo_id as i64; if let Some(p) = prefix { - sqlx::query("UPDATE rt_nodes SET prefix = $1 WHERE id = $2") + sqlx::query("UPDATE rt_nodes SET prefix = $1 WHERE repo_id = $2 AND id = $3") .bind(p) + .bind(rid) .bind(id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; } if let Some(r) = record { - sqlx::query("UPDATE rt_nodes SET record = $1 WHERE id = $2") + sqlx::query("UPDATE rt_nodes SET record = $1 WHERE repo_id = $2 AND id = $3") .bind(r as i64) + .bind(rid) .bind(id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; } Ok(()) } async fn get_node(&self, id: usize) -> Result<(Vec, usize)> { - let row = sqlx::query_as::<_, (Vec, i64)>("SELECT prefix, record FROM rt_nodes WHERE id = $1") - .bind(id as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (Vec, i64)>( + "SELECT prefix, record FROM rt_nodes WHERE repo_id = $1 AND id = $2", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; let Some((prefix, record)) = row else { return Err(StorageError::BranchOutOfRange(id)); }; @@ -112,32 +165,40 @@ impl Storage for PostgresStorage { } async fn get_children(&self, id: usize) -> Result> { - let rows = sqlx::query_as::<_, (i64,)>("SELECT child FROM rt_children WHERE parent = $1 ORDER BY child") - .bind(id as i64) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let rows = sqlx::query_as::<_, (i64,)>( + "SELECT child FROM rt_children WHERE repo_id = $1 AND parent = $2 ORDER BY child", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; Ok(rows.into_iter().map(|(c,)| c as usize).collect()) } async fn set_root(&mut self, shard: usize, root: usize) -> Result<()> { sqlx::query( - "INSERT INTO rt_roots (shard, root) VALUES ($1, $2) ON CONFLICT (shard) DO UPDATE SET root = EXCLUDED.root", + "INSERT INTO rt_roots (repo_id, shard, root) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, shard) DO UPDATE SET root = EXCLUDED.root", ) - .bind(shard as i64) + .bind(self.repo_id as i64) + .bind(shard as i32) .bind(root as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_root(&self, shard: usize) -> Result { - let row = sqlx::query_as::<_, (i64,)>("SELECT root FROM rt_roots WHERE shard = $1") - .bind(shard as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (i64,)>( + "SELECT root FROM rt_roots WHERE repo_id = $1 AND shard = $2", + ) + .bind(self.repo_id as i64) + .bind(shard as i32) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; let Some((root,)) = row else { return Err(StorageError::BranchOutOfRange(shard)); }; @@ -146,114 +207,746 @@ impl Storage for PostgresStorage { async fn set_meta(&mut self, record: usize, meta: &[u8]) -> Result<()> { sqlx::query( - "INSERT INTO rt_meta (record, meta) VALUES ($1, $2) ON CONFLICT (record) DO UPDATE SET meta = EXCLUDED.meta", + "INSERT INTO rt_meta (repo_id, record, meta) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, record) DO UPDATE SET meta = EXCLUDED.meta", ) + .bind(self.repo_id as i64) .bind(record as i64) .bind(meta) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_meta(&self, record: usize) -> Result>> { - let row = sqlx::query_as::<_, (Vec,)>("SELECT meta FROM rt_meta WHERE record = $1") - .bind(record as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT meta FROM rt_meta WHERE repo_id = $1 AND record = $2", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; Ok(row.map(|(m,)| m)) } async fn set_key_len(&mut self, record: usize, len: usize) -> Result<()> { sqlx::query( - "INSERT INTO rt_keylen (record, len) VALUES ($1, $2) ON CONFLICT (record) DO UPDATE SET len = EXCLUDED.len", + "INSERT INTO rt_keylen (repo_id, record, len) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, record) DO UPDATE SET len = EXCLUDED.len", ) + .bind(self.repo_id as i64) .bind(record as i64) - .bind(len as i64) + .bind(len as i32) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_key_len(&self, record: usize) -> Result> { - let row = sqlx::query_as::<_, (i64,)>("SELECT len FROM rt_keylen WHERE record = $1") - .bind(record as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (i32,)>( + "SELECT len FROM rt_keylen WHERE repo_id = $1 AND record = $2", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; Ok(row.map(|(len,)| len as usize)) } async fn add_shortcut_node(&mut self, shard: usize, elem: &[u8], node_id: usize) -> Result<()> { sqlx::query( - "INSERT INTO rt_shortcuts (shard, elem, node_id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + "INSERT INTO rt_shortcuts (repo_id, shard, elem, node_id) VALUES ($1, $2, $3, $4) \ + ON CONFLICT DO NOTHING", ) - .bind(shard as i64) + .bind(self.repo_id as i64) + .bind(shard as i32) .bind(elem) .bind(node_id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_shortcut_nodes(&self, shard: usize, elem: &[u8]) -> Result> { - let rows = sqlx::query_as::<_, (i64,)>("SELECT node_id FROM rt_shortcuts WHERE shard = $1 AND elem = $2") - .bind(shard as i64) - .bind(elem) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let rows = sqlx::query_as::<_, (i64,)>( + "SELECT node_id FROM rt_shortcuts WHERE repo_id = $1 AND shard = $2 AND elem = $3", + ) + .bind(self.repo_id as i64) + .bind(shard as i32) + .bind(elem) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; Ok(rows.into_iter().map(|(id,)| id as usize).collect()) } async fn clear_shortcuts(&mut self) -> Result<()> { - sqlx::query("DELETE FROM rt_shortcuts") + sqlx::query("DELETE FROM rt_shortcuts WHERE repo_id = $1") + .bind(self.repo_id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn set_edge_data(&mut self, edge: usize, data: &[u8]) -> Result<()> { sqlx::query( - "INSERT INTO rt_edges (id, data) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data", + "INSERT INTO rt_edges (repo_id, id, data) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, id) DO UPDATE SET data = EXCLUDED.data", ) + .bind(self.repo_id as i64) .bind(edge as i64) .bind(data) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; Ok(()) } async fn get_edge_data(&self, edge: usize) -> Result>> { - let row = sqlx::query_as::<_, (Vec,)>("SELECT data FROM rt_edges WHERE id = $1") - .bind(edge as i64) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT data FROM rt_edges WHERE repo_id = $1 AND id = $2", + ) + .bind(self.repo_id as i64) + .bind(edge as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; Ok(row.map(|(d,)| d)) } async fn clear_edges(&mut self) -> Result<()> { - sqlx::query("DELETE FROM rt_edges") + sqlx::query("DELETE FROM rt_edges WHERE repo_id = $1") + .bind(self.repo_id as i64) .execute(&self.pool) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(db_err)?; + Ok(()) + } + + async fn for_each_edge_data( + &self, + f: &mut (dyn for<'a> FnMut(usize, &'a [u8]) -> Result<()> + Send), + ) -> Result<()> { + let rows = sqlx::query("SELECT id, data FROM rt_edges WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + for r in &rows { + let id: i64 = r.try_get("id").map_err(db_err)?; + let data: Vec = r.try_get("data").map_err(db_err)?; + f(id as usize, &data)?; + } + Ok(()) + } + + async fn set_node_meta(&mut self, elem: usize, meta: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_node_meta (repo_id, elem, meta) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, elem) DO UPDATE SET meta = EXCLUDED.meta", + ) + .bind(self.repo_id as i64) + .bind(elem as i64) + .bind(meta) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn get_node_meta(&self, elem: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT meta FROM rt_node_meta WHERE repo_id = $1 AND elem = $2", + ) + .bind(self.repo_id as i64) + .bind(elem as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(m,)| m)) + } + + async fn clear_node_meta(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_node_meta WHERE repo_id = $1") + .bind(self.repo_id as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn set_chain(&mut self, record: usize, chain: &[u64]) -> Result<()> { + let bytes = encode_chain(chain); + sqlx::query( + "INSERT INTO rt_chains (repo_id, record, chain) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, record) DO UPDATE SET chain = EXCLUDED.chain", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .bind(bytes) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn get_chain(&self, record: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT chain FROM rt_chains WHERE repo_id = $1 AND record = $2", + ) + .bind(self.repo_id as i64) + .bind(record as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| decode_chain(&b))) + } + + async fn clear_chains(&mut self) -> Result<()> { + sqlx::query("DELETE FROM rt_chains WHERE repo_id = $1") + .bind(self.repo_id as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn save_symbol(&mut self, sym: &Symbol) -> Result<()> { + let annotations = serde_json::to_string(&sym.annotations).map_err(ser_err)?; + sqlx::query( + "INSERT INTO sg_symbols \ + (repo_id, id, name, kind, scope, scope_id, type_ref, type_name, file, \ + line, end_line, signature, doc, annotations, language) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) \ + ON CONFLICT (repo_id, id) DO UPDATE SET \ + name = EXCLUDED.name, kind = EXCLUDED.kind, scope = EXCLUDED.scope, \ + scope_id = EXCLUDED.scope_id, type_ref = EXCLUDED.type_ref, \ + type_name = EXCLUDED.type_name, file = EXCLUDED.file, line = EXCLUDED.line, \ + end_line = EXCLUDED.end_line, signature = EXCLUDED.signature, doc = EXCLUDED.doc, \ + annotations = EXCLUDED.annotations, language = EXCLUDED.language", + ) + .bind(self.repo_id as i64) + .bind(sym.id as i64) + .bind(&sym.name) + .bind(sym.kind.as_str()) + .bind(sym.scope.as_str()) + .bind(sym.scope_id as i64) + .bind(sym.type_ref as i64) + .bind(&sym.type_name) + .bind(&sym.file) + .bind(sym.line as i32) + .bind(sym.end_line as i32) + .bind(&sym.signature) + .bind(&sym.doc) + .bind(annotations) + .bind(&sym.language) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_symbol(&self, id: u64) -> Result> { + let row = sqlx::query( + "SELECT id, name, kind, scope, scope_id, type_ref, type_name, file, line, \ + end_line, signature, doc, annotations, language \ + FROM sg_symbols WHERE repo_id = $1 AND id = $2", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.as_ref().map(row_to_symbol).transpose()?) + } + + async fn load_all_symbols(&self) -> Result> { + let rows = sqlx::query( + "SELECT id, name, kind, scope, scope_id, type_ref, type_name, file, line, \ + end_line, signature, doc, annotations, language FROM sg_symbols WHERE repo_id = $1", + ) + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.iter().map(row_to_symbol).collect() + } + + async fn save_next_id(&mut self, next: u64) -> Result<()> { + sqlx::query( + "INSERT INTO sg_next_id (repo_id, next) VALUES ($1, $2) \ + ON CONFLICT (repo_id) DO UPDATE SET next = EXCLUDED.next", + ) + .bind(self.repo_id as i64) + .bind(next as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_next_id(&self) -> Result { + let row: Option<(i64,)> = + sqlx::query_as("SELECT next FROM sg_next_id WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(v,)| v as u64).unwrap_or(0)) + } + + async fn all_chains(&self) -> Result)>> { + let rows = sqlx::query("SELECT record, chain FROM rt_chains WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + let record: i64 = r.try_get("record").map_err(db_err)?; + let chain: Vec = r.try_get("chain").map_err(db_err)?; + out.push((record as u64, chain)); + } + Ok(out) + } + + async fn set_call_records(&mut self, func: u64, records: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO sg_call_records (repo_id, func, records) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, func) DO UPDATE SET records = EXCLUDED.records", + ) + .bind(self.repo_id as i64) + .bind(func as i64) + .bind(records) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn get_call_records(&self, func: u64) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT records FROM sg_call_records WHERE repo_id = $1 AND func = $2", + ) + .bind(self.repo_id as i64) + .bind(func as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| b)) + } + + async fn all_call_records(&self) -> Result)>> { + let rows = sqlx::query("SELECT func, records FROM sg_call_records WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + let func: i64 = r.try_get("func").map_err(db_err)?; + let records: Vec = r.try_get("records").map_err(db_err)?; + out.push((func as u64, records)); + } + Ok(out) + } + + async fn set_call_name_index(&mut self, name: &str, sites: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO sg_call_names (repo_id, name, sites) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, name) DO UPDATE SET sites = EXCLUDED.sites", + ) + .bind(self.repo_id as i64) + .bind(name) + .bind(sites) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_call_name_index(&self, name: &str) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT sites FROM sg_call_names WHERE repo_id = $1 AND name = $2", + ) + .bind(self.repo_id as i64) + .bind(name) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| b)) + } + + async fn all_call_name_indexes(&self) -> Result)>> { + let rows = sqlx::query("SELECT name, sites FROM sg_call_names WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + let name: String = r.try_get("name").map_err(db_err)?; + let sites: Vec = r.try_get("sites").map_err(db_err)?; + out.push((name, sites)); + } + Ok(out) + } + + async fn upsert_file(&mut self, f: &FileInfo) -> Result<()> { + sqlx::query( + "INSERT INTO sg_files (repo_id, path, language, bytes, lines) VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (repo_id, path) DO UPDATE SET \ + language = EXCLUDED.language, bytes = EXCLUDED.bytes, lines = EXCLUDED.lines", + ) + .bind(self.repo_id as i64) + .bind(&f.path) + .bind(&f.language) + .bind(f.bytes as i64) + .bind(f.lines as i32) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_all_files(&self) -> Result> { + let rows = sqlx::query("SELECT path, language, bytes, lines FROM sg_files WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut out = Vec::with_capacity(rows.len()); + for r in &rows { + out.push(FileInfo { + path: r.try_get("path").map_err(db_err)?, + language: r.try_get("language").map_err(db_err)?, + bytes: r.try_get::("bytes").map_err(db_err)? as u64, + lines: r.try_get::("lines").map_err(db_err)? as u32, + }); + } + Ok(out) + } + + async fn version(&self) -> Result { + let row: Option<(i64,)> = sqlx::query_as("SELECT version FROM sg_meta WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(v,)| v as u64).unwrap_or(0)) + } + + async fn set_version(&mut self, v: u64) -> Result<()> { + sqlx::query( + "INSERT INTO sg_meta (repo_id, version) VALUES ($1, $2) \ + ON CONFLICT (repo_id) DO UPDATE SET version = EXCLUDED.version", + ) + .bind(self.repo_id as i64) + .bind(v as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn clear_entities(&mut self) -> Result<()> { + let rid = self.repo_id as i64; + let mut tx = self.pool.begin().await.map_err(db_err)?; + for t in [ + "sg_symbols", + "sg_files", + "sg_call_records", + "sg_call_names", + "rt_nodes", + "rt_children", + "rt_roots", + "rt_meta", + "rt_keylen", + "rt_shortcuts", + "rt_chains", + "rt_edges", + "rt_node_meta", + "rt_node_blooms", + ] { + sqlx::query(&format!("DELETE FROM {t} WHERE repo_id = $1")) + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + sqlx::query("UPDATE rt_counter SET next = 1 WHERE repo_id = $1") + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query("UPDATE sg_next_id SET next = 100 WHERE repo_id = $1") + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query("UPDATE sg_meta SET version = 0 WHERE repo_id = $1") + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query( + "INSERT INTO rt_nodes (repo_id, id, prefix, record) VALUES ($1, 0, '', 0) \ + ON CONFLICT (repo_id, id) DO NOTHING", + ) + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; Ok(()) } - // The remaining methods are either no‑ops or can be forwarded to other - // storage implementations if needed. For now we keep the minimal set. - async fn set_node_meta(&mut self, _elem: usize, _meta: &[u8]) -> Result<()> { Ok(()) } - async fn get_node_meta(&self, _elem: usize) -> Result>> { Ok(None) } - async fn clear_node_meta(&mut self) -> Result<()> { Ok(()) } - async fn set_chain(&mut self, _record: usize, _chain: &[u64]) -> Result<()> { Ok(()) } - async fn get_chain(&self, _record: usize) -> Result>> { Ok(None) } - async fn clear_chains(&mut self) -> Result<()> { Ok(()) } - async fn save_symbol(&mut self, _sym: &codegraph_core::Symbol) -> Result<()> { Ok(()) } - async fn load_symbol(&self, _id: u64) -> Result> { Ok(None) } + #[cfg(feature = "bloom-search")] + async fn set_node_bloom(&mut self, id: usize, bloom: &[u8]) -> Result<()> { + sqlx::query( + "INSERT INTO rt_node_blooms (repo_id, id, bloom) VALUES ($1, $2, $3) \ + ON CONFLICT (repo_id, id) DO UPDATE SET bloom = EXCLUDED.bloom", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .bind(bloom) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + #[cfg(feature = "bloom-search")] + async fn get_node_bloom(&self, id: usize) -> Result>> { + let row = sqlx::query_as::<_, (Vec,)>( + "SELECT bloom FROM rt_node_blooms WHERE repo_id = $1 AND id = $2", + ) + .bind(self.repo_id as i64) + .bind(id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.map(|(b,)| b)) + } + + fn new_tx(&self) -> Box { + Box::new(PostgresTx { + pool: self.pool.clone(), + repo_id: self.repo_id, + nodes: Vec::new(), + ops: Vec::new(), + }) + } +} + +/// Probe version index trên đĩa (dùng cho `SharedGraphIndex::ensure_fresh`) — +/// không mở toàn bộ index. `None`/lỗi → coi như version 0. +#[cfg(feature = "postgres")] +impl PostgresStorage { + pub async fn probe_version(dsn: &str, repo_id: u64) -> Result { + let pool = PgPoolOptions::new() + .max_connections(2) + .connect(dsn) + .await + .map_err(db_err)?; + let row: Option<(i64,)> = + sqlx::query_as("SELECT version FROM sg_meta WHERE repo_id = $1") + .bind(repo_id as i64) + .fetch_optional(&pool) + .await + .map_err(db_err)?; + Ok(row.map(|(v,)| v as u64).unwrap_or(0)) + } +} + +// ==================== PostgresTx ==================== + +/// Transaction cho `PostgresStorage`: buffer mutation, áp dụng atomic trong 1 +/// Postgres transaction tại `commit`. `new_node` cấp id nguyên tử per-repo ngay +/// lúc reservation (tránh trùng id khi nhiều writer). +pub struct PostgresTx { + pool: PgPool, + repo_id: u64, + nodes: Vec<(usize, Vec, usize)>, + ops: Vec, +} + +#[async_trait] +impl Tx for PostgresTx { + async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { + let row: (i64,) = sqlx::query_as( + "UPDATE rt_counter SET next = next + 1 WHERE repo_id = $1 RETURNING next - 1", + ) + .bind(self.repo_id as i64) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + let id = row.0 as usize; + self.nodes.push((id, prefix, record)); + Ok(id) + } + + async fn update_node( + &mut self, + id: usize, + prefix: Option>, + record: Option, + ) -> Result<()> { + self.ops.push(super::TxOp::UpdateNode { id, prefix, record }); + Ok(()) + } + + async fn add_child(&mut self, parent: usize, child: usize) -> Result<()> { + self.ops.push(super::TxOp::AddChild { parent, child }); + Ok(()) + } + + async fn move_child(&mut self, from: usize, to: usize, child: usize) -> Result<()> { + self.ops.push(super::TxOp::MoveChild { from, to, child }); + Ok(()) + } + + async fn commit(self: Box) -> Result<()> { + let PostgresTx { + pool, + repo_id, + nodes, + ops, + } = *self; + let rid = repo_id as i64; + let mut tx = pool.begin().await.map_err(db_err)?; + + for (id, prefix, record) in &nodes { + sqlx::query( + "INSERT INTO rt_nodes (repo_id, id, prefix, record) VALUES ($1, $2, $3, $4) \ + ON CONFLICT (repo_id, id) DO NOTHING", + ) + .bind(rid) + .bind(*id as i64) + .bind(prefix) + .bind(*record as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + + if let Some(max_id) = nodes.iter().map(|(id, _, _)| *id).max() { + sqlx::query("UPDATE rt_counter SET next = GREATEST(next, $1) WHERE repo_id = $2") + .bind((max_id + 1) as i64) + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + + for op in ops { + match op { + super::TxOp::AddChild { parent, child } => { + sqlx::query( + "INSERT INTO rt_children (repo_id, parent, child) VALUES ($1, $2, $3) \ + ON CONFLICT DO NOTHING", + ) + .bind(rid) + .bind(parent as i64) + .bind(child as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + super::TxOp::MoveChild { from, to, child } => { + sqlx::query( + "DELETE FROM rt_children WHERE repo_id = $1 AND parent = $2 AND child = $3", + ) + .bind(rid) + .bind(from as i64) + .bind(child as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query( + "INSERT INTO rt_children (repo_id, parent, child) VALUES ($1, $2, $3) \ + ON CONFLICT DO NOTHING", + ) + .bind(rid) + .bind(to as i64) + .bind(child as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + super::TxOp::UpdateNode { id, prefix, record } => { + if let Some(p) = prefix { + sqlx::query( + "UPDATE rt_nodes SET prefix = $1 WHERE repo_id = $2 AND id = $3", + ) + .bind(p) + .bind(rid) + .bind(id as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + if let Some(r) = record { + sqlx::query( + "UPDATE rt_nodes SET record = $1 WHERE repo_id = $2 AND id = $3", + ) + .bind(r as i64) + .bind(rid) + .bind(id as i64) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + } + } + } + + tx.commit().await.map_err(db_err)?; + Ok(()) + } +} + +/// Map một row `sg_symbols` → `codegraph_core::Symbol`. +fn row_to_symbol(row: &PgRow) -> Result { + let id: i64 = row.try_get("id").map_err(db_err)?; + let name: String = row.try_get("name").map_err(db_err)?; + let kind: String = row.try_get("kind").map_err(db_err)?; + let scope: String = row.try_get("scope").map_err(db_err)?; + let scope_id: i64 = row.try_get("scope_id").map_err(db_err)?; + let type_ref: i64 = row.try_get("type_ref").map_err(db_err)?; + let type_name: Option = row.try_get("type_name").map_err(db_err)?; + let file: String = row.try_get("file").map_err(db_err)?; + let line: i32 = row.try_get("line").map_err(db_err)?; + let end_line: i32 = row.try_get("end_line").map_err(db_err)?; + let signature: Option = row.try_get("signature").map_err(db_err)?; + let doc: Option = row.try_get("doc").map_err(db_err)?; + let annotations: String = row.try_get("annotations").map_err(db_err)?; + let language: String = row.try_get("language").map_err(db_err)?; + let kind = SymbolKind::parse(&kind) + .ok_or_else(|| StorageError::Internal(format!("bad symbol kind: {kind}")))?; + let scope = ScopeLevel::parse(&scope) + .ok_or_else(|| StorageError::Internal(format!("bad scope level: {scope}")))?; + let annotations: Vec = serde_json::from_str(&annotations).map_err(ser_err)?; + Ok(Symbol { + id: id as u64, + name, + kind, + scope, + scope_id: scope_id as u64, + type_ref: type_ref as u64, + type_name, + file, + line: line as u32, + end_line: end_line as u32, + signature, + doc, + annotations, + language, + }) } diff --git a/crates/codegraph-graph/tests/rdbms.rs b/crates/codegraph-graph/tests/rdbms.rs new file mode 100644 index 000000000..7fd8c390c --- /dev/null +++ b/crates/codegraph-graph/tests/rdbms.rs @@ -0,0 +1,163 @@ +//! Integration tests cho RDBMS backend (Postgres/MySQL, feature `postgres` / +//! `mysql`) — multi-tenant + sharding. +//! +//! Chỉ chạy khi có DB thật: đặt `TEST_RDBMS_DSN` (`postgres://...` hoặc +//! `mysql://...`) và `TEST_RDBMS_REPO_ID` (u64), rồi bật feature + bỏ ignore: +//! +//! ```sh +//! TEST_RDBMS_DSN=postgres://user:pass@localhost:5432/codegraph \ +//! TEST_RDBMS_REPO_ID=123 \ +//! cargo test -p codegraph-graph --features postgres --test rdbms -- --ignored +//! ``` +//! +//! Schema (`sql//001` + `002`) phải đã được apply thủ công lên server +//! trước (migration không chạy tự động). Test mặc định bị `#[ignore]` nên không +//! ảnh hưởng `cargo test` thường. + +#![cfg(any(feature = "postgres", feature = "mysql"))] + +use codegraph_core::{CallRecord, EffectType, ScopeLevel, SYMBOL_BASE, Symbol, SymbolKind}; +use codegraph_graph::{GraphIndex, ParseResult}; +use codegraph_core::StorageRoute; +use std::collections::HashMap; + +fn sym(file: &str, name: &str, id: u64) -> Symbol { + Symbol { + id, + name: name.to_string(), + kind: SymbolKind::Function, + scope: ScopeLevel::Global, + scope_id: 0, + type_ref: 0, + type_name: None, + file: file.to_string(), + line: 1, + end_line: 1, + signature: None, + doc: None, + annotations: Vec::new(), + language: "ts".to_string(), + } +} + +fn result( + path: &str, + symbols: Vec, + chains: HashMap>, + calls: Vec, +) -> ParseResult { + ParseResult { + path: path.to_string(), + language: "ts".to_string(), + bytes: 0, + lines: 0, + symbols, + chains, + calls, + } +} + +/// Ingest → reopen trên backend RDBMS: entity sống lại từ partition `repo_id`, +/// query surface (symbols/chains/edges/files) khớp, version bump đúng. +#[tokio::test] +#[ignore = "requires a running Postgres/MySQL; set TEST_RDBMS_DSN + TEST_RDBMS_REPO_ID"] +async fn rdbms_ingest_reopen_roundtrip() { + let dsn = match std::env::var("TEST_RDBMS_DSN") { + Ok(d) => d, + Err(_) => { + eprintln!("skip: TEST_RDBMS_DSN not set"); + return; + } + }; + let repo_id: u64 = std::env::var("TEST_RDBMS_REPO_ID") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let route = StorageRoute::Sharded { + dsns: vec![dsn], + repo_id: Some(repo_id), + root: None, + }; + + let calls = vec![CallRecord { + caller_id: SYMBOL_BASE, + call_name: "b".to_string(), + position: 1, + arg_exprs: vec!["x".to_string()], + line: 3, + condition: None, + is_loop_body: false, + effect: EffectType::None, + effect_desc: None, + target_class: None, + target_method: None, + }]; + let r = result( + "a.ts", + vec![ + sym("a.ts", "a", SYMBOL_BASE), + sym("a.ts", "b", SYMBOL_BASE + 1), + ], + HashMap::from([(SYMBOL_BASE, vec![SYMBOL_BASE, SYMBOL_BASE + 1])]), + calls, + ); + + { + let mut idx = GraphIndex::open_route(&route).await.expect("open rdbms"); + idx.ingest(&[r]).await.expect("ingest"); + assert_eq!(idx.version(), 1); + } + + // Reopen cùng repo_id → query lại được toàn bộ từ partition. + let idx = GraphIndex::open_route(&route).await.expect("reopen rdbms"); + assert_eq!(idx.version(), 1); + assert_eq!(idx.stats().symbols, 2); + assert_eq!(idx.stats().chains, 1); + assert_eq!(idx.stats().edges, 1); + assert_eq!(idx.files().len(), 1); + assert_eq!(idx.files()[0].path, "a.ts"); + + let callees = idx.callees(SYMBOL_BASE).await.unwrap(); + assert_eq!(callees.len(), 1); + assert_eq!(callees[0].name, "b"); +} + +/// Ingest rỗng = full wipe trên partition `repo_id`: entity cũ biến mất, version +/// vẫn bump (như sqlite/lmdb). +#[tokio::test] +#[ignore = "requires a running Postgres/MySQL; set TEST_RDBMS_DSN + TEST_RDBMS_REPO_ID"] +async fn rdbms_empty_ingest_wipes_store() { + let dsn = match std::env::var("TEST_RDBMS_DSN") { + Ok(d) => d, + Err(_) => { + eprintln!("skip: TEST_RDBMS_DSN not set"); + return; + } + }; + let repo_id: u64 = std::env::var("TEST_RDBMS_REPO_ID") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let route = StorageRoute::Sharded { + dsns: vec![dsn], + repo_id: Some(repo_id), + root: None, + }; + + let r = result( + "a.ts", + vec![sym("a.ts", "a", SYMBOL_BASE)], + HashMap::from([(SYMBOL_BASE, vec![SYMBOL_BASE])]), + vec![], + ); + let mut idx = GraphIndex::open_route(&route).await.expect("open rdbms"); + idx.ingest(&[r]).await.expect("ingest"); + assert_eq!(idx.stats().symbols, 1); + + idx.ingest(&[]).await.expect("empty ingest"); + assert_eq!(idx.version(), 2); + assert_eq!(idx.stats().symbols, 0); + assert!(idx.symbol_by_id(SYMBOL_BASE).is_none()); +} diff --git a/crates/codegraph-graph/tests/redis.rs b/crates/codegraph-graph/tests/redis.rs new file mode 100644 index 000000000..45ac55bb3 --- /dev/null +++ b/crates/codegraph-graph/tests/redis.rs @@ -0,0 +1,147 @@ +//! Integration tests cho Redis backend (feature `redis`). +//! +//! Chỉ chạy khi có Redis thật: đặt `TEST_REDIS_DSN` (ví dụ +//! `redis://127.0.0.1:6379`) rồi bật feature + bỏ ignore: +//! +//! ```sh +//! TEST_REDIS_DSN=redis://127.0.0.1:6379 \ +//! cargo test -p codegraph-graph --features redis --test redis -- --ignored +//! ``` +//! +//! Keyspace prefix được dẫn xuất từ số DB trong DSN (`/15` → `codegraph:idx:15`), +//! nên test này (DB mặc định 0) không đụng hàng với unit test nội bộ (DB 15). +//! Test mặc định bị `#[ignore]` nên không ảnh hưởng `cargo test` thường. + +#![cfg(feature = "redis")] + +use codegraph_core::{ + CallRecord, EffectType, ScopeLevel, StorageRoute, SYMBOL_BASE, Symbol, SymbolKind, +}; +use codegraph_graph::{GraphIndex, ParseResult}; +use std::collections::HashMap; + +fn sym(file: &str, name: &str, id: u64) -> Symbol { + Symbol { + id, + name: name.to_string(), + kind: SymbolKind::Function, + scope: ScopeLevel::Global, + scope_id: 0, + type_ref: 0, + type_name: None, + file: file.to_string(), + line: 1, + end_line: 1, + signature: None, + doc: None, + annotations: Vec::new(), + language: "ts".to_string(), + } +} + +fn result( + path: &str, + symbols: Vec, + chains: HashMap>, + calls: Vec, +) -> ParseResult { + ParseResult { + path: path.to_string(), + language: "ts".to_string(), + bytes: 0, + lines: 0, + symbols, + chains, + calls, + } +} + +/// Ingest → reopen trên Redis: entity sống lại từ keyspace, query surface +/// (symbols/chains/edges/files) khớp, version bump đúng. +#[tokio::test] +#[ignore = "requires a running Redis; set TEST_REDIS_DSN"] +async fn redis_ingest_reopen_roundtrip() { + let dsn = match std::env::var("TEST_REDIS_DSN") { + Ok(d) => d, + Err(_) => { + eprintln!("skip: TEST_REDIS_DSN not set"); + return; + } + }; + + let calls = vec![CallRecord { + caller_id: SYMBOL_BASE, + call_name: "b".to_string(), + position: 1, + arg_exprs: vec!["x".to_string()], + line: 3, + condition: None, + is_loop_body: false, + effect: EffectType::None, + effect_desc: None, + target_class: None, + target_method: None, + }]; + let r = result( + "a.ts", + vec![ + sym("a.ts", "a", SYMBOL_BASE), + sym("a.ts", "b", SYMBOL_BASE + 1), + ], + HashMap::from([(SYMBOL_BASE, vec![SYMBOL_BASE, SYMBOL_BASE + 1])]), + calls, + ); + + { + let mut idx = GraphIndex::open_route(&StorageRoute::Local(dsn.clone())) + .await + .expect("open redis"); + idx.ingest(&[r]).await.expect("ingest"); + assert_eq!(idx.version(), 1); + } + + // Reopen cùng keyspace → query lại được toàn bộ. + let idx = GraphIndex::open_route(&StorageRoute::Local(dsn)) + .await + .expect("reopen redis"); + assert_eq!(idx.version(), 1); + assert_eq!(idx.stats().symbols, 2); + assert_eq!(idx.stats().chains, 1); + assert_eq!(idx.stats().edges, 1); + assert_eq!(idx.files().len(), 1); + assert_eq!(idx.files()[0].path, "a.ts"); + + let callees = idx.callees(SYMBOL_BASE).await.unwrap(); + assert_eq!(callees.len(), 1); + assert_eq!(callees[0].name, "b"); +} + +/// Ingest rỗng = full wipe trên Redis: entity cũ biến mất, version vẫn bump. +#[tokio::test] +#[ignore = "requires a running Redis; set TEST_REDIS_DSN"] +async fn redis_empty_ingest_wipes_store() { + let dsn = match std::env::var("TEST_REDIS_DSN") { + Ok(d) => d, + Err(_) => { + eprintln!("skip: TEST_REDIS_DSN not set"); + return; + } + }; + + let r = result( + "a.ts", + vec![sym("a.ts", "a", SYMBOL_BASE)], + HashMap::from([(SYMBOL_BASE, vec![SYMBOL_BASE])]), + vec![], + ); + let mut idx = GraphIndex::open_route(&StorageRoute::Local(dsn)) + .await + .expect("open redis"); + idx.ingest(&[r]).await.expect("ingest"); + assert_eq!(idx.stats().symbols, 1); + + idx.ingest(&[]).await.expect("empty ingest"); + assert_eq!(idx.version(), 2); + assert_eq!(idx.stats().symbols, 0); + assert!(idx.symbol_by_id(SYMBOL_BASE).is_none()); +} diff --git a/crates/codegraph-mcp/Cargo.toml b/crates/codegraph-mcp/Cargo.toml index 6cdf30218..70d45c1aa 100644 --- a/crates/codegraph-mcp/Cargo.toml +++ b/crates/codegraph-mcp/Cargo.toml @@ -9,6 +9,9 @@ repository.workspace = true # Luồng HTTP MCP riêng (session theo mcp-session-id): dùng rmcp # `transport-streamable-http-server` + axum để mount StreamableHttpService. http = ["rmcp/transport-streamable-http-server", "dep:axum"] +# Backend RDBMS (Postgres/MySQL, multi-tenant + sharding). Bật để MCP server +# (và CLI `codegraph serve`) có thể mở index trên Postgres/MySQL. +rdbms = ["codegraph-graph/postgres", "codegraph-graph/mysql"] [dependencies] codegraph-api = { path = "../codegraph-api" } diff --git a/crates/codegraph-mcp/src/session.rs b/crates/codegraph-mcp/src/session.rs index 8ad681e51..29260ef74 100644 --- a/crates/codegraph-mcp/src/session.rs +++ b/crates/codegraph-mcp/src/session.rs @@ -17,6 +17,7 @@ use anyhow::{anyhow, Result}; use camino::{Utf8Path, Utf8PathBuf}; +use codegraph_core::StorageRoute; use codegraph_extract::{init_project, project_dir, ExtractConfig, ExtractStats, Orchestrator}; use codegraph_graph::{GraphIndex, SharedGraphIndex}; use serde_json::{json, Value}; @@ -93,7 +94,7 @@ enum SessionState { Empty, /// Đã bind vào một workspace root, storage + index dùng chung sẵn sàng. Ready { - dsn: Option, + route: Option, shared_index: Arc, }, } @@ -144,9 +145,11 @@ impl Session { /// `with_root()` nhưng seed sẵn output format từ CLI lúc khởi động. pub async fn with_root_and_format(root: Utf8PathBuf, format: OutputStyle) -> Result { let state = if project_dir(&root).exists() { - let dsn = ExtractConfig::load(&root).storage_dsn(&root); - let shared_index = Arc::new(SharedGraphIndex::open(dsn.clone()).await?); - SessionState::Ready { dsn, shared_index } + // RDBMS cần repo_id — đảm bảo đã sinh (self-heal) trước khi tính route. + let _ = ExtractConfig::ensure_repo_id(&root); + let route = ExtractConfig::load(&root).storage_route(&root); + let shared_index = Arc::new(SharedGraphIndex::open_route(route.clone()).await?); + SessionState::Ready { route, shared_index } } else { SessionState::Empty }; @@ -190,25 +193,28 @@ impl Session { ) -> Result { let root = normalize_root(path)?; let dir = init_project(&root)?; + // RDBMS backend (postgres/mysql) cần `repo_id` làm partition key — + // sinh ngẫu nhiên rồi ghi vào config nếu thiếu (self-heal). + let _ = ExtractConfig::ensure_repo_id(&root); let indexed = if do_index { Some(run_index(&root).await?) } else { None }; - // Config giờ đã tồn tại → load đúng backend (sqlite/lmdb/redis/...). - let dsn = ExtractConfig::load(&root).storage_dsn(&root); - let shared_index = Arc::new(SharedGraphIndex::open(dsn.clone()).await?); + // Config giờ đã tồn tại → load đúng backend (sqlite/lmdb/redis/rdbms/...). + let route = ExtractConfig::load(&root).storage_route(&root); + let shared_index = Arc::new(SharedGraphIndex::open_route(route.clone()).await?); // Root set trước state — mọi `ensure_ready` đồng thời đọc root mới sẽ - // tự swap state theo DSN mới (xem `ensure_ready`). + // tự swap state theo route mới (xem `ensure_ready`). *self.root.write().await = Some(root.clone()); *self.detail.write().await = detail; if let Some(f) = format { *self.format.write().await = f; } let mut st = self.state.write().await; - *st = SessionState::Ready { dsn, shared_index }; + *st = SessionState::Ready { route, shared_index }; Ok(InitOutcome { root, dir, indexed }) } @@ -252,28 +258,30 @@ impl Session { codegraph_index {{}} to build the index." )); } - let dsn = ExtractConfig::load(&root).storage_dsn(&root); + // RDBMS cần repo_id — đảm bảo đã sinh (self-heal) trước khi tính route. + let _ = ExtractConfig::ensure_repo_id(&root); + let route = ExtractConfig::load(&root).storage_route(&root); let mut st = self.state.write().await; // Root được init giữa chừng (vd sau khi init() lỗi part-way) → chuyển // từ Empty sang Ready bằng cách load storage. let was_empty = matches!(&*st, SessionState::Empty); if was_empty { - let shared_index = Arc::new(SharedGraphIndex::open(dsn.clone()).await?); - *st = SessionState::Ready { dsn, shared_index }; + let shared_index = Arc::new(SharedGraphIndex::open_route(route.clone()).await?); + *st = SessionState::Ready { route, shared_index }; } else if let SessionState::Ready { - dsn: cur, + route: cur, shared_index, } = &mut *st { // Config đổi backend giữa chừng → load lại storage. - if *cur != dsn { - match SharedGraphIndex::open(dsn.clone()).await { + if *cur != route { + match SharedGraphIndex::open_route(route.clone()).await { Ok(sgi) => { *shared_index = Arc::new(sgi); - *cur = dsn; + *cur = route; } - Err(e) => eprintln!("[codegraph] open index for {dsn:?} failed: {e}"), + Err(e) => eprintln!("[codegraph] open index for {route:?} failed: {e}"), } } } @@ -326,8 +334,10 @@ fn normalize_root(path: Utf8PathBuf) -> Result { /// (ingest = full re-index, bump version → snapshot cũ bị `ensure_fresh` thấy /// stale và rebuild ở lần query kế). async fn run_index(root: &Utf8Path) -> Result { - let mut idx = match ExtractConfig::load(root).storage_dsn(root) { - Some(dsn) => GraphIndex::open(&dsn).await?, + // RDBMS cần repo_id (partition key) — sinh nếu thiếu trước khi mở index. + let _ = ExtractConfig::ensure_repo_id(root); + let mut idx = match ExtractConfig::load(root).storage_route(root) { + Some(route) => GraphIndex::open_route(&route).await?, None => GraphIndex::in_memory(), }; Orchestrator::with_registry() diff --git a/crates/codegraph/Cargo.toml b/crates/codegraph/Cargo.toml index ec5bc50c8..ba58cc99d 100644 --- a/crates/codegraph/Cargo.toml +++ b/crates/codegraph/Cargo.toml @@ -26,4 +26,7 @@ camino = { workspace = true } indicatif = "0.18.6" [features] -default = [] +# Mặc định bật RDBMS (Postgres/MySQL) để CLI + MCP server có thể serve backend +# multi-tenant. Tắt để build nhẹ: `cargo build --no-default-features`. +default = ["rdbms"] +rdbms = ["codegraph-graph/postgres", "codegraph-graph/mysql", "codegraph-mcp/rdbms"] diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 084d10b3f..2b288a42c 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -169,15 +169,18 @@ async fn cmd_default(_root: &Utf8Path) -> Result<()> { } /// DSN (kèm scheme) của backend storage trong config — `None` = in-memory. +/// Chỉ dùng cho watcher (cần DSN string); RDBMS trả `None` (watcher không spawn). fn storage_dsn(root: &Utf8Path) -> Option { codegraph_extract::ExtractConfig::load(root).storage_dsn(root) } -/// Mở index theo backend đã config (DSN scheme → `GraphIndex::open`). +/// Mở index theo backend đã config (`StorageRoute` → `GraphIndex::open_route`). async fn open_index(root: &Utf8Path) -> Result { - // `.codegraph/` đã được init (có config) — lúc này storage dsn đã biết. - match storage_dsn(root) { - Some(dsn) => Ok(GraphIndex::open(&dsn).await?), + // `.codegraph/` đã được init (có config) — lúc này storage route đã biết. + // RDBMS cần repo_id (partition key) — sinh nếu thiếu (self-heal). + let _ = codegraph_extract::ExtractConfig::ensure_repo_id(root); + match codegraph_extract::ExtractConfig::load(root).storage_route(root) { + Some(route) => Ok(GraphIndex::open_route(&route).await?), None => Ok(GraphIndex::in_memory()), } } @@ -187,6 +190,8 @@ async fn open_index(root: &Utf8Path) -> Result { async fn cmd_init(root: &Utf8Path, do_index: bool, show_progress: bool) -> Result<()> { let dir = codegraph_extract::init_project(root)?; eprintln!("initialized {}", dir); + // RDBMS cần repo_id (partition key) — sinh ngẫu nhiên nếu thiếu (self-heal). + let _ = codegraph_extract::ExtractConfig::ensure_repo_id(root); if do_index { let stats = index_all(root, show_progress).await?; diff --git a/sql/README.md b/sql/README.md index d8c172dd7..660503d45 100644 --- a/sql/README.md +++ b/sql/README.md @@ -38,9 +38,10 @@ sql/ ### 1. Partition theo repository -- Mọi bảng dữ liệu dẫn đầu bằng cột `repo_id VARCHAR(64) NOT NULL` — là **UUID** - sinh lúc `codegraph init`, lưu trong `.codegraph/config.toml` (`[storage] - repo_id = "…"`). Một project root (`.codegraph/`) = một repository. +- Mọi bảng dữ liệu dẫn đầu bằng cột `repo_id BIGINT NOT NULL` — là **số u64** + sinh ngẫu nhiên lúc `codegraph init`, lưu trong `.codegraph/config.toml` + (`[storage] repo_id = `). Một project root (`.codegraph/`) = một + repository. - PK composite `(repo_id, …)` trên mọi bảng → các repo cô lập hoàn toàn; re-index / xoá một repo chỉ là `DELETE … WHERE repo_id = ?`. - `repo_id` nằm trong **handle của backend** (thuộc `Storage` impl), không đụng @@ -115,11 +116,11 @@ giữ cột bytea/blob; vẫn partition theo `repo_id`): | case-sensitivity | chính xác theo byte | `COLLATE utf8mb4_bin` để giữ case-sensitive cho name/path | | composite key `rt_shortcuts` | PK `(repo_id, shard, elem, node_id)` | `elem LONGBLOB` không vào PK được → index `elem(255)` prefix + PK không gồm elem (xem ghi chú) | -> Ghi chú MySQL về giới hạn key: index key tối đa 3072 bytes (utf8mb4 → 768 ký -> tự). `repo_id VARCHAR(64)` + `name/file/path VARCHAR(700)` = 764 ký tự × 4 = -> 3056 bytes — vừa đủ. Giá trị dài hơn 700 ký tự cần hash key (md5/sha256) -> ở phase sau; schema hiện tại chấp nhận giới hạn này (tên call / path thực tế -> hiếm khi vượt). +> Ghi chú MySQL về giới hạn key: index key tối đa 3072 bytes (utf8mb4 → 4 byte/ +> ký tự). `repo_id BIGINT` (8 bytes) + `name/file/path VARCHAR(700)` (700 × 4 = +> 2800 bytes) ≈ 2808 bytes — nằm dưới 3072, vừa đủ. Giá trị dài hơn 700 ký tự +> cần hash key (md5/sha256) ở phase sau; schema hiện tại chấp nhận giới hạn này +> (tên call / path thực tế hiếm khi vượt). > > `rt_shortcuts.elem` là bytes nhị phân (element id encode) có thể rất dài → > MySQL dùng prefix index `elem(255)`; Postgres giữ PK đầy đủ. Vì lookup luôn From c0f6aed550ca4a127a3fc900568d5b70569c658b Mon Sep 17 00:00:00 2001 From: hungpham10 <136320753+hungpham10@users.noreply.github.com> Date: Sat, 15 Aug 2026 09:50:16 +0000 Subject: [PATCH 3/9] style: apply rustfmt --- crates/codegraph-core/src/route.rs | 4 +- crates/codegraph-extract/src/config.rs | 6 +-- crates/codegraph-graph/src/lib.rs | 37 ++++++++-------- crates/codegraph-graph/src/shared.rs | 4 +- crates/codegraph-graph/src/storage/mysql.rs | 42 ++++++++++--------- .../codegraph-graph/src/storage/postgres.rs | 40 +++++++++--------- crates/codegraph-graph/tests/rdbms.rs | 4 +- crates/codegraph-graph/tests/redis.rs | 2 +- crates/codegraph-mcp/src/http.rs | 1 - crates/codegraph-mcp/src/session.rs | 15 +++++-- crates/codegraph/src/main.rs | 9 +++- 11 files changed, 91 insertions(+), 73 deletions(-) diff --git a/crates/codegraph-core/src/route.rs b/crates/codegraph-core/src/route.rs index 6285add31..e3c2de9ec 100644 --- a/crates/codegraph-core/src/route.rs +++ b/crates/codegraph-core/src/route.rs @@ -7,8 +7,7 @@ /// /// `PartialEq` dùng để session/MCP so sánh route hiện tại với route mới khi root /// đổi (`ensure_ready` swap index nếu khác). -#[derive(Debug, Clone, PartialEq, Eq)] -#[derive(Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub enum StorageRoute { /// In-memory (test/dev, không persist). #[default] @@ -65,4 +64,3 @@ impl StorageRoute { } } } - diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index a3386d92e..f112e624a 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -194,11 +194,7 @@ impl ExtractConfig { }) } StorageKind::Sqlite | StorageKind::Lmdb | StorageKind::Redis => { - let dsn = self - .storage - .dsn - .clone() - .or_else(|| self.storage_dsn(root)); + let dsn = self.storage.dsn.clone().or_else(|| self.storage_dsn(root)); Some(StorageRoute::Local(dsn?)) } } diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 4391a5dd9..e271bfa6e 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -38,13 +38,13 @@ pub use crate::search::Search; use crate::search::SearchResume; #[cfg(feature = "lmdb")] pub use crate::storage::lmdb::LmdbStorage; +#[cfg(feature = "mysql")] +pub use crate::storage::mysql::MySqlStorage; +#[cfg(feature = "postgres")] +pub use crate::storage::postgres::PostgresStorage; #[cfg(feature = "sqlite")] pub use crate::storage::sqlite::SqliteStorage; pub use crate::storage::{InMemoryStorage, Storage, Tx}; -#[cfg(feature = "postgres")] -pub use crate::storage::postgres::PostgresStorage; -#[cfg(feature = "mysql")] -pub use crate::storage::mysql::MySqlStorage; use codegraph_core::{ CallRecord, CallSite, CallSiteResult, ClassInfo, DependenciesReport, Dependency, EdgeMeta, EffectType, Error, FileInfo, FlowCall, FlowResult, FunctionScope, MemberInfo, ResolveResult, @@ -368,16 +368,20 @@ impl GraphIndex { .trim_start_matches("redis://") .rsplit('/') .next() - .and_then(|s| if s.is_empty() { None } else { s.parse::().ok() }) + .and_then(|s| { + if s.is_empty() { + None + } else { + s.parse::().ok() + } + }) .unwrap_or(0); - let client = redis::Client::open(dsn) - .map_err(|e| Error::Db(format!("redis client: {e}")))?; - let storage = crate::storage::redis::RedisStorage::new( - client, - &format!("codegraph:idx:{db}"), - ) - .await - .map_err(serr)?; + let client = + redis::Client::open(dsn).map_err(|e| Error::Db(format!("redis client: {e}")))?; + let storage = + crate::storage::redis::RedisStorage::new(client, &format!("codegraph:idx:{db}")) + .await + .map_err(serr)?; let storage = Arc::new(RwLock::new(storage)) as Arc>; let mut idx = Self::new_with_storage(storage); idx.rebuild().await?; @@ -408,9 +412,9 @@ impl GraphIndex { let shard = route.shard_of(repo_id).ok_or_else(|| { Error::Db("không tính được shard từ StorageRoute::Sharded".into()) })?; - let dsn = dsns.get(shard).ok_or_else(|| { - Error::Db(format!("shard {shard} vượt quá số lượng DSN")) - })?; + let dsn = dsns + .get(shard) + .ok_or_else(|| Error::Db(format!("shard {shard} vượt quá số lượng DSN")))?; let result: Result = if dsn.starts_with("postgres://") { #[cfg(feature = "postgres")] { @@ -459,7 +463,6 @@ impl GraphIndex { } } - fn new_with_storage(storage: Arc>) -> Self { // Name engine luôn in-memory (như semgraph SearchIndex) — storage riêng // để record id (1..N) không đụng record của chain engine (func ids). diff --git a/crates/codegraph-graph/src/shared.rs b/crates/codegraph-graph/src/shared.rs index cd3358f81..7eb9cbfd4 100644 --- a/crates/codegraph-graph/src/shared.rs +++ b/crates/codegraph-graph/src/shared.rs @@ -127,7 +127,9 @@ impl SharedGraphIndex { Some(StorageRoute::Local(d)) => d.as_str(), _ => return None, }; - crate::storage::lmdb::probe_version(trim_scheme(dsn)).await.ok() + crate::storage::lmdb::probe_version(trim_scheme(dsn)) + .await + .ok() } #[cfg(feature = "postgres")] "postgres" => { diff --git a/crates/codegraph-graph/src/storage/mysql.rs b/crates/codegraph-graph/src/storage/mysql.rs index 21074cb3d..2ca22f3db 100644 --- a/crates/codegraph-graph/src/storage/mysql.rs +++ b/crates/codegraph-graph/src/storage/mysql.rs @@ -1,8 +1,8 @@ +use super::{Result, Storage, StorageError, Tx, decode_chain, encode_chain}; use async_trait::async_trait; +use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; use sqlx::mysql::{MySqlPoolOptions, MySqlRow}; use sqlx::{MySqlPool, Row}; -use super::{decode_chain, encode_chain, Result, Storage, StorageError, Tx}; -use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; /// MySQL implementation của `Storage` trait — multi-tenant (mọi bảng dẫn đầu bằng /// `repo_id`), theo thiết kế `sql/README.md`. Instance được bind vào một `repo_id`. @@ -314,7 +314,7 @@ impl Storage for MySqlStorage { .bind(self.repo_id as i64) .execute(&self.pool) .await - .map_err(db_err)?; + .map_err(db_err)?; Ok(()) } @@ -366,7 +366,7 @@ impl Storage for MySqlStorage { .bind(self.repo_id as i64) .execute(&self.pool) .await - .map_err(db_err)?; + .map_err(db_err)?; Ok(()) } @@ -402,7 +402,7 @@ impl Storage for MySqlStorage { .bind(self.repo_id as i64) .execute(&self.pool) .await - .map_err(db_err)?; + .map_err(db_err)?; Ok(()) } @@ -535,7 +535,7 @@ impl Storage for MySqlStorage { .bind(self.repo_id as i64) .fetch_all(&self.pool) .await - .map_err(db_err)?; + .map_err(db_err)?; let mut out = Vec::with_capacity(rows.len()); for r in &rows { let func: i64 = r.try_get("func").map_err(db_err)?; @@ -576,7 +576,7 @@ impl Storage for MySqlStorage { .bind(self.repo_id as i64) .fetch_all(&self.pool) .await - .map_err(db_err)?; + .map_err(db_err)?; let mut out = Vec::with_capacity(rows.len()); for r in &rows { let name: String = r.try_get("name").map_err(db_err)?; @@ -604,11 +604,12 @@ impl Storage for MySqlStorage { } async fn load_all_files(&self) -> Result> { - let rows = sqlx::query("SELECT path, language, bytes, lines FROM sg_files WHERE repo_id = ?") - .bind(self.repo_id as i64) - .fetch_all(&self.pool) - .await - .map_err(db_err)?; + let rows = + sqlx::query("SELECT path, language, bytes, lines FROM sg_files WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; let mut out = Vec::with_capacity(rows.len()); for r in &rows { out.push(FileInfo { @@ -626,7 +627,7 @@ impl Storage for MySqlStorage { .bind(self.repo_id as i64) .fetch_optional(&self.pool) .await - .map_err(db_err)?; + .map_err(db_err)?; Ok(row.map(|(v,)| v as u64).unwrap_or(0)) } @@ -683,11 +684,13 @@ impl Storage for MySqlStorage { .execute(&mut *tx) .await .map_err(db_err)?; - sqlx::query("INSERT IGNORE INTO rt_nodes (repo_id, id, prefix, record) VALUES (?, 0, '', 0)") - .bind(rid) - .execute(&mut *tx) - .await - .map_err(db_err)?; + sqlx::query( + "INSERT IGNORE INTO rt_nodes (repo_id, id, prefix, record) VALUES (?, 0, '', 0)", + ) + .bind(rid) + .execute(&mut *tx) + .await + .map_err(db_err)?; tx.commit().await.map_err(db_err)?; Ok(()) } @@ -784,7 +787,8 @@ impl Tx for MySqlTx { prefix: Option>, record: Option, ) -> Result<()> { - self.ops.push(super::TxOp::UpdateNode { id, prefix, record }); + self.ops + .push(super::TxOp::UpdateNode { id, prefix, record }); Ok(()) } diff --git a/crates/codegraph-graph/src/storage/postgres.rs b/crates/codegraph-graph/src/storage/postgres.rs index f6b0d74a9..0692902f4 100644 --- a/crates/codegraph-graph/src/storage/postgres.rs +++ b/crates/codegraph-graph/src/storage/postgres.rs @@ -1,8 +1,8 @@ +use super::{Result, Storage, StorageError, Tx, decode_chain, encode_chain}; use async_trait::async_trait; +use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; use sqlx::postgres::{PgPoolOptions, PgRow}; use sqlx::{PgPool, Row}; -use super::{decode_chain, encode_chain, Result, Storage, StorageError, Tx}; -use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; /// PostgreSQL implementation của `Storage` trait — multi-tenant theo thiết kế /// `sql/README.md`: mọi bảng dẫn đầu bằng `repo_id`, 1 repository = 1 partition. @@ -492,12 +492,11 @@ impl Storage for PostgresStorage { } async fn load_next_id(&self) -> Result { - let row: Option<(i64,)> = - sqlx::query_as("SELECT next FROM sg_next_id WHERE repo_id = $1") - .bind(self.repo_id as i64) - .fetch_optional(&self.pool) - .await - .map_err(db_err)?; + let row: Option<(i64,)> = sqlx::query_as("SELECT next FROM sg_next_id WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; Ok(row.map(|(v,)| v as u64).unwrap_or(0)) } @@ -616,11 +615,12 @@ impl Storage for PostgresStorage { } async fn load_all_files(&self) -> Result> { - let rows = sqlx::query("SELECT path, language, bytes, lines FROM sg_files WHERE repo_id = $1") - .bind(self.repo_id as i64) - .fetch_all(&self.pool) - .await - .map_err(db_err)?; + let rows = + sqlx::query("SELECT path, language, bytes, lines FROM sg_files WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; let mut out = Vec::with_capacity(rows.len()); for r in &rows { out.push(FileInfo { @@ -755,12 +755,11 @@ impl PostgresStorage { .connect(dsn) .await .map_err(db_err)?; - let row: Option<(i64,)> = - sqlx::query_as("SELECT version FROM sg_meta WHERE repo_id = $1") - .bind(repo_id as i64) - .fetch_optional(&pool) - .await - .map_err(db_err)?; + let row: Option<(i64,)> = sqlx::query_as("SELECT version FROM sg_meta WHERE repo_id = $1") + .bind(repo_id as i64) + .fetch_optional(&pool) + .await + .map_err(db_err)?; Ok(row.map(|(v,)| v as u64).unwrap_or(0)) } } @@ -798,7 +797,8 @@ impl Tx for PostgresTx { prefix: Option>, record: Option, ) -> Result<()> { - self.ops.push(super::TxOp::UpdateNode { id, prefix, record }); + self.ops + .push(super::TxOp::UpdateNode { id, prefix, record }); Ok(()) } diff --git a/crates/codegraph-graph/tests/rdbms.rs b/crates/codegraph-graph/tests/rdbms.rs index 7fd8c390c..5010b1ede 100644 --- a/crates/codegraph-graph/tests/rdbms.rs +++ b/crates/codegraph-graph/tests/rdbms.rs @@ -16,9 +16,9 @@ #![cfg(any(feature = "postgres", feature = "mysql"))] -use codegraph_core::{CallRecord, EffectType, ScopeLevel, SYMBOL_BASE, Symbol, SymbolKind}; -use codegraph_graph::{GraphIndex, ParseResult}; use codegraph_core::StorageRoute; +use codegraph_core::{CallRecord, EffectType, SYMBOL_BASE, ScopeLevel, Symbol, SymbolKind}; +use codegraph_graph::{GraphIndex, ParseResult}; use std::collections::HashMap; fn sym(file: &str, name: &str, id: u64) -> Symbol { diff --git a/crates/codegraph-graph/tests/redis.rs b/crates/codegraph-graph/tests/redis.rs index 45ac55bb3..673d67362 100644 --- a/crates/codegraph-graph/tests/redis.rs +++ b/crates/codegraph-graph/tests/redis.rs @@ -15,7 +15,7 @@ #![cfg(feature = "redis")] use codegraph_core::{ - CallRecord, EffectType, ScopeLevel, StorageRoute, SYMBOL_BASE, Symbol, SymbolKind, + CallRecord, EffectType, SYMBOL_BASE, ScopeLevel, StorageRoute, Symbol, SymbolKind, }; use codegraph_graph::{GraphIndex, ParseResult}; use std::collections::HashMap; diff --git a/crates/codegraph-mcp/src/http.rs b/crates/codegraph-mcp/src/http.rs index 7dbc8acea..26ee37fa8 100644 --- a/crates/codegraph-mcp/src/http.rs +++ b/crates/codegraph-mcp/src/http.rs @@ -80,7 +80,6 @@ pub async fn serve_http( // Deprecated original serve_http – replaced by extended version with observability and auth support. // The old implementation has been removed to avoid duplicate symbol definitions. - /// Smoke test: POST `initialize` qua tower oneshot (không cần TCP) → HTTP /// 200 + response SSE chứa `serverInfo.name = codegraph`. Module này chỉ /// compile khi feature `http` bật (lib.rs gate toàn bộ `mod http`). diff --git a/crates/codegraph-mcp/src/session.rs b/crates/codegraph-mcp/src/session.rs index 29260ef74..16e3d178a 100644 --- a/crates/codegraph-mcp/src/session.rs +++ b/crates/codegraph-mcp/src/session.rs @@ -149,7 +149,10 @@ impl Session { let _ = ExtractConfig::ensure_repo_id(&root); let route = ExtractConfig::load(&root).storage_route(&root); let shared_index = Arc::new(SharedGraphIndex::open_route(route.clone()).await?); - SessionState::Ready { route, shared_index } + SessionState::Ready { + route, + shared_index, + } } else { SessionState::Empty }; @@ -214,7 +217,10 @@ impl Session { *self.format.write().await = f; } let mut st = self.state.write().await; - *st = SessionState::Ready { route, shared_index }; + *st = SessionState::Ready { + route, + shared_index, + }; Ok(InitOutcome { root, dir, indexed }) } @@ -268,7 +274,10 @@ impl Session { let was_empty = matches!(&*st, SessionState::Empty); if was_empty { let shared_index = Arc::new(SharedGraphIndex::open_route(route.clone()).await?); - *st = SessionState::Ready { route, shared_index }; + *st = SessionState::Ready { + route, + shared_index, + }; } else if let SessionState::Ready { route: cur, shared_index, diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 2b288a42c..63e2eb4bf 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -268,7 +268,14 @@ async fn cmd_serve( if use_root && is_initialized(root) { watcher::spawn(root.to_path_buf(), storage_dsn(root)); } - return codegraph_mcp::serve_http(format, addr, allowed_hosts, enable_observability, api_key).await; + return codegraph_mcp::serve_http( + format, + addr, + allowed_hosts, + enable_observability, + api_key, + ) + .await; } if !mcp { return Err(anyhow!( From 1794b0f12cee7c71bca14d869493445ac0a94760 Mon Sep 17 00:00:00 2001 From: hungpham10 <136320753+hungpham10@users.noreply.github.com> Date: Sat, 15 Aug 2026 17:11:21 +0700 Subject: [PATCH 4/9] Potential fix for pull request finding 'CodeQL / Workflow does not contain permissions' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/integration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 745e95504..f080e9040 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,5 +1,8 @@ name: Integration (Postgres / MySQL / Redis) +permissions: + contents: read + # Chạy test tích hợp trên backend thật (Postgres/MySQL/Redis) qua service # container của GitHub Actions. Schema được apply thủ công (`sql//*`) # trước khi chạy test — khớp thiết kế "migration thủ công" của repo. From 86333f7075313cd0a31e2fdadf4ea452af809889 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 15 Aug 2026 18:23:24 +0700 Subject: [PATCH 5/9] Add tests --- crates/codegraph-api/src/lib.rs | 411 ++++++++++++++++-- crates/codegraph-api/tests/api.rs | 271 +++++++++++- crates/codegraph-graph/src/lib.rs | 175 +++++++- .../codegraph-mcp/src/server-instructions.md | 8 +- crates/codegraph-mcp/src/tools.rs | 218 ++++++++-- 5 files changed, 985 insertions(+), 98 deletions(-) diff --git a/crates/codegraph-api/src/lib.rs b/crates/codegraph-api/src/lib.rs index cf1387be5..641044604 100644 --- a/crates/codegraph-api/src/lib.rs +++ b/crates/codegraph-api/src/lib.rs @@ -21,8 +21,43 @@ pub struct GraphApi { sessions: Arc, } +/// Giá trị `timeout_ms` đặc biệt: deadline **đã hết hạn ngay tại thời điểm gọi** +/// → search chắc chắn `timed_out` trên mọi máy (dùng cho test xác định, không +/// phụ thuộc tốc độ đồng hồ tường như `timeout_ms = 1`). +pub const TIMEOUT_EXPIRE_IMMEDIATELY: u64 = u64::MAX; + // ==================== Search session store ==================== +/// Loại search tạo resume — dùng validate resume id (không cho cross-tool +/// resume: id của `codegraph_search` không dùng được cho `codegraph_references`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResumeKind { + Name, + Annotation, + ListKind, + References, + Flow, +} + +/// Mô tả query lưu trong resume để validate: tool-type + query + kind phải +/// khớp. Sai → lỗi bảo LLM retry không có `resume`. +#[derive(Debug, Clone)] +pub struct ResumeDesc { + pub ty: ResumeKind, + pub query: String, + pub kind: Option, +} + +/// Cursor resume lưu trong session store. +/// - `Name`: name search (DFS checkpoint từ engine). +/// - `Offset`: các search scan tuyến tính (annotation/list_by_kind/references/ +/// flow) — tiếp tục từ `next` offset; `desc` để validate resume. +#[derive(Debug, Clone)] +pub enum ResumeCursor { + Name(SearchCursor), + Offset { next: usize, desc: ResumeDesc }, +} + /// Cursor session lưu **phía server** — LLM chỉ cầm một id ngắn (hex) và echo /// lại khi retry. Id vô nghĩa ngoài tiến trình này: index version đổi (re-ingest) /// hoặc server restart → session stale, báo LLM retry không có `resume`. @@ -30,7 +65,7 @@ struct StoredResume { created: Instant, /// Version index lúc tạo — đổi (re-ingest) → cursor mất giá trị. index_version: u64, - cursor: SearchCursor, + cursor: ResumeCursor, } /// Store in-process cho resume id → cursor. Không persist; purge theo TTL khi @@ -60,7 +95,7 @@ impl SearchSessionStore { /// Lưu cursor, trả id hex ngắn. Trước khi thêm: purge session quá TTL, chặn /// số session tối đa (evict session già nhất). - pub fn put(&self, cursor: SearchCursor, index_version: u64) -> String { + pub fn put(&self, cursor: ResumeCursor, index_version: u64) -> String { let mut map = self.inner.lock().unwrap(); let now = Instant::now(); map.retain(|_, s| now.duration_since(s.created) < self.ttl); @@ -88,7 +123,7 @@ impl SearchSessionStore { } /// Đọc cursor theo id — `None` nếu không có / quá TTL. - pub fn get(&self, id: &str) -> Option<(u64, SearchCursor)> { + pub fn get(&self, id: &str) -> Option<(u64, ResumeCursor)> { let map = self.inner.lock().unwrap(); map.get(id).map(|s| (s.index_version, s.cursor.clone())) } @@ -127,6 +162,30 @@ pub struct ResumeSearchOutcome { pub index_version: u64, } +/// Kết quả resumable cho search trả `Vec` (`codegraph_references` +/// / `codegraph_search_by_call`). Cùng hình dạng [`ResumeSearchOutcome`]. +#[derive(Debug)] +pub struct ResumeCallSiteOutcome { + pub page: Vec, + pub timed_out: bool, + /// Số kết quả đã collect lúc ngắt (dùng cho message báo LLM). + pub progress: usize, + /// Resume id để retry khi `timed_out`; `None` khi hoàn tất. + pub resume: Option, + pub index_version: u64, +} + +/// Kết quả resumable cho search trả `Vec` +/// (`codegraph_search_flow`). Cùng hình dạng [`ResumeSearchOutcome`]. +#[derive(Debug)] +pub struct ResumeFlowOutcome { + pub page: Vec, + pub timed_out: bool, + pub progress: usize, + pub resume: Option, + pub index_version: u64, +} + /// Phân trang cho search symbol: `limit` chặn số symbol mỗi trang (`0` = /// không giới hạn), `offset` bỏ qua `offset` symbol đầu. #[derive(Debug, Clone, Copy)] @@ -170,6 +229,8 @@ impl GraphApi { /// Resumable + deadline-aware của [`Self::search`] — nền cho /// `codegraph_search`. `timeout_ms = 0` = không giới hạn thời gian. + /// `timeout_ms = u64::MAX` ([`TIMEOUT_EXPIRE_IMMEDIATELY`]) = deadline đã + /// hết hạn ngay → chắc chắn `timed_out` (dùng cho test xác định). /// `resume` = id trả về từ lần timeout trước (phải cùng query). pub async fn search_resumable( &self, @@ -206,7 +267,9 @@ impl GraphApi { } /// Resumable + deadline-aware của [`Self::search_symbol_paged`] — nền cho - /// `codegraph_search_symbol`. `timeout_ms = 0` = không giới hạn. + /// `codegraph_search_symbol`. `timeout_ms = 0` = không giới hạn; + /// `timeout_ms = u64::MAX` ([`TIMEOUT_EXPIRE_IMMEDIATELY`]) = chắc chắn + /// `timed_out` (dùng cho test xác định). /// /// `resume` được validate (index version + query/mode/kind phải khớp) — /// sai → lỗi báo LLM retry không có `resume`. @@ -235,21 +298,35 @@ impl GraphApi { .into(), )); } - if stored.query != q || stored.mode != mode || stored.kind != kind { + let c = match stored { + ResumeCursor::Name(c) => c, + _ => { + return Err(Error::Invalid( + "resume id was created for a different query — retry without resume" + .into(), + )) + } + }; + if c.query != q || c.mode != mode || c.kind != kind { return Err(Error::Invalid( "resume id was created for a different query — retry without resume".into(), )); } - Some(stored) + Some(c) } None => None, }; // ── Deadline ── - let deadline = if timeout_ms == 0 { - None - } else { - Some(Instant::now() + Duration::from_millis(timeout_ms)) + // `timeout_ms == 0` → không giới hạn (None) + // `timeout_ms == u64::MAX` → [`TIMEOUT_EXPIRE_IMMEDIATELY`]: deadline + // đã hết hạn ngay → chắc chắn timed_out + // (dùng cho test xác định) + // khác → now + timeout_ms + let deadline = match timeout_ms { + 0 => None, + u64::MAX => Some(Instant::now()), + _ => Some(Instant::now() + Duration::from_millis(timeout_ms)), }; let out = idx @@ -269,7 +346,7 @@ impl GraphApi { // ── Quản lý session: lưu khi còn tiếp tục (timeout / còn page), xoá // khi xong hẳn. ── let resume_id = match &out.cursor { - Some(c) => Some(self.sessions.put(c.clone(), version)), + Some(c) => Some(self.sessions.put(ResumeCursor::Name(c.clone()), version)), None => { if let Some(id) = &resume { self.sessions.remove(id); @@ -368,31 +445,7 @@ impl GraphApi { /// symbol (resolve exact — trùng tên lấy ứng viên đầu). pub async fn search_flow_pattern(&self, pattern: &str) -> Result> { let idx = self.index().await; - let mut ids = Vec::new(); - for tok in pattern.split(',') { - let t = tok.trim(); - if t.is_empty() { - continue; - } - if let Ok(n) = t.parse::() { - ids.push(n); - continue; - } - if let Some(m) = codegraph_core::marker_id(t) { - ids.push(m); - continue; - } - let r = idx.resolve_by_name_or_id(t, 0)?; - let sid = r - .symbol - .map(|s| s.id) - .or_else(|| r.matches.first().map(|s| s.id)) - .ok_or_else(|| Error::Invalid(format!("unknown flow token: {t}")))?; - ids.push(sid); - } - if ids.is_empty() { - return Err(Error::Invalid("empty flow pattern".into())); - } + let ids = resolve_flow_pattern_ids(&idx, pattern)?; idx.search_flow(&ids).await } @@ -404,6 +457,250 @@ impl GraphApi { .await } + /// Validate resume id (nếu có) cho các search scan tuyến tính (Offset cursor): + /// index version + tool-type + query + kind phải khớp. Trả `Some(offset)` để + /// tiếp tục, hoặc `None` (không resume → caller dùng `pagination.offset`). + fn resolve_offset( + &self, + resume: &Option, + version: u64, + ty: ResumeKind, + q: &str, + kind: Option, + ) -> Result> { + match resume { + Some(id) => { + let (stored_version, stored) = self.sessions.get(id).ok_or_else(|| { + Error::Invalid("resume id expired or unknown — retry without resume".into()) + })?; + if stored_version != version { + return Err(Error::Invalid( + "index was re-built since this resume was created — retry without resume" + .into(), + )); + } + match stored { + ResumeCursor::Offset { next, desc } => { + if desc.ty != ty || desc.query != q || desc.kind != kind { + return Err(Error::Invalid( + "resume id was created for a different query — retry without resume" + .into(), + )); + } + Ok(Some(next)) + } + _ => Err(Error::Invalid( + "resume id was created for a different query — retry without resume".into(), + )), + } + } + None => Ok(None), + } + } + + /// Resumable + deadline-aware của [`Self::search_by_annotation`]. `timeout_ms` + /// như [`Self::search_symbol_paged_resumable`] (0 = không giới hạn, `u64::MAX` + /// = chắc chắn timed_out). `resume` validate (index version + annotation + + /// kind phải khớp) — sai → lỗi bảo LLM retry không có `resume`. + pub async fn search_by_annotation_resumable( + &self, + annotation: &str, + kind: Option, + pagination: Pagination, + resume: Option, + timeout_ms: u64, + ) -> Result { + let idx = self.index().await; + let version = idx.version(); + let q = annotation.to_lowercase(); + let offset = self + .resolve_offset(&resume, version, ResumeKind::Annotation, &q, kind)? + .unwrap_or(pagination.offset as usize); + let deadline = deadline_from(timeout_ms); + let (page, total, cont) = idx.search_by_annotation_resumable( + annotation, + kind, + offset, + pagination.limit as usize, + deadline, + ); + let timed_out = cont.is_some(); + let progress = page.len(); + let resume_id = if timed_out { + Some(self.sessions.put( + ResumeCursor::Offset { + next: offset, + desc: ResumeDesc { + ty: ResumeKind::Annotation, + query: q, + kind, + }, + }, + version, + )) + } else { + if let Some(id) = &resume { + self.sessions.remove(id); + } + None + }; + Ok(ResumeSearchOutcome { + page, + total, + timed_out, + progress, + resume: resume_id, + index_version: version, + }) + } + + /// Resumable + deadline-aware của [`Self::list_by_kind`]. `timeout_ms` như + /// [`Self::search_symbol_paged_resumable`]. `resume` validate (index version + /// + kind phải khớp). + pub async fn list_by_kind_resumable( + &self, + kind: SymbolKind, + pagination: Pagination, + resume: Option, + timeout_ms: u64, + ) -> Result { + let idx = self.index().await; + let version = idx.version(); + let offset = self + .resolve_offset(&resume, version, ResumeKind::ListKind, "", Some(kind))? + .unwrap_or(pagination.offset as usize); + let deadline = deadline_from(timeout_ms); + let (page, total, cont) = + idx.list_symbols_by_kind_resumable(kind, offset, pagination.limit as usize, deadline); + let timed_out = cont.is_some(); + let progress = page.len(); + let resume_id = if timed_out { + Some(self.sessions.put( + ResumeCursor::Offset { + next: offset, + desc: ResumeDesc { + ty: ResumeKind::ListKind, + query: String::new(), + kind: Some(kind), + }, + }, + version, + )) + } else { + if let Some(id) = &resume { + self.sessions.remove(id); + } + None + }; + Ok(ResumeSearchOutcome { + page, + total, + timed_out, + progress, + resume: resume_id, + index_version: version, + }) + } + + /// Resumable + deadline-aware của [`Self::references`]. `timeout_ms` như + /// [`Self::search_symbol_paged_resumable`]. `resume` validate (index version + /// + query phải khớp). + pub async fn references_resumable( + &self, + query: &str, + pagination: Pagination, + resume: Option, + timeout_ms: u64, + ) -> Result { + let idx = self.index().await; + let version = idx.version(); + let q = query.to_lowercase(); + let offset = self + .resolve_offset(&resume, version, ResumeKind::References, &q, None)? + .unwrap_or(pagination.offset as usize); + let deadline = deadline_from(timeout_ms); + let (page, cont) = idx + .callers_by_call_name_resumable(query, offset, pagination.limit as usize, deadline) + .await?; + let timed_out = cont.is_some(); + let progress = page.len(); + let resume_id = if timed_out { + Some(self.sessions.put( + ResumeCursor::Offset { + next: offset, + desc: ResumeDesc { + ty: ResumeKind::References, + query: q, + kind: None, + }, + }, + version, + )) + } else { + if let Some(id) = &resume { + self.sessions.remove(id); + } + None + }; + Ok(ResumeCallSiteOutcome { + page, + timed_out, + progress, + resume: resume_id, + index_version: version, + }) + } + + /// Resumable + deadline-aware của [`Self::search_flow_pattern`]. `timeout_ms` + /// như [`Self::search_symbol_paged_resumable`]. `resume` validate (index + /// version + pattern phải khớp). + pub async fn search_flow_pattern_resumable( + &self, + pattern: &str, + pagination: Pagination, + resume: Option, + timeout_ms: u64, + ) -> Result { + let idx = self.index().await; + let version = idx.version(); + let q = pattern.to_lowercase(); + let offset = self + .resolve_offset(&resume, version, ResumeKind::Flow, &q, None)? + .unwrap_or(pagination.offset as usize); + let ids = resolve_flow_pattern_ids(&idx, pattern)?; + let deadline = deadline_from(timeout_ms); + let (page, cont) = idx + .search_flow_resumable(&ids, offset, pagination.limit as usize, deadline) + .await?; + let timed_out = cont.is_some(); + let progress = page.len(); + let resume_id = if timed_out { + Some(self.sessions.put( + ResumeCursor::Offset { + next: offset, + desc: ResumeDesc { + ty: ResumeKind::Flow, + query: q, + kind: None, + }, + }, + version, + )) + } else { + if let Some(id) = &resume { + self.sessions.remove(id); + } + None + }; + Ok(ResumeFlowOutcome { + page, + timed_out, + progress, + resume: resume_id, + index_version: version, + }) + } + pub async fn context_markdown(&self, req: &ContextRequest) -> Result { codegraph_context::build(&self.shared_index, req).await } @@ -425,3 +722,45 @@ impl GraphApi { self.index().await.stats() } } + +/// Deadline từ `timeout_ms`: `0` = không giới hạn (None), `u64::MAX` +/// ([`TIMEOUT_EXPIRE_IMMEDIATELY`]) = đã hết hạn ngay (chắc chắn timed_out), +/// khác = `now + timeout_ms`. +fn deadline_from(timeout_ms: u64) -> Option { + match timeout_ms { + 0 => None, + u64::MAX => Some(Instant::now()), + _ => Some(Instant::now() + Duration::from_millis(timeout_ms)), + } +} + +/// Resolve pattern string thành danh sách id (số / marker / tên symbol) — dùng +/// chung cho [`GraphApi::search_flow_pattern`] và bản resumable. +fn resolve_flow_pattern_ids(idx: &GraphIndex, pattern: &str) -> Result> { + let mut ids = Vec::new(); + for tok in pattern.split(',') { + let t = tok.trim(); + if t.is_empty() { + continue; + } + if let Ok(n) = t.parse::() { + ids.push(n); + continue; + } + if let Some(m) = codegraph_core::marker_id(t) { + ids.push(m); + continue; + } + let r = idx.resolve_by_name_or_id(t, 0)?; + let sid = r + .symbol + .map(|s| s.id) + .or_else(|| r.matches.first().map(|s| s.id)) + .ok_or_else(|| Error::Invalid(format!("unknown flow token: {t}")))?; + ids.push(sid); + } + if ids.is_empty() { + return Err(Error::Invalid("empty flow pattern".into())); + } + Ok(ids) +} diff --git a/crates/codegraph-api/tests/api.rs b/crates/codegraph-api/tests/api.rs index 7f791203a..1c8e8cbc3 100644 --- a/crates/codegraph-api/tests/api.rs +++ b/crates/codegraph-api/tests/api.rs @@ -3,7 +3,7 @@ use codegraph_core::{ CallRecord, EffectType, ScopeLevel, Symbol, SymbolKind, SymbolMatch, SYMBOL_BASE, }; use codegraph_graph::{GraphIndex, ParseResult, SharedGraphIndex}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; fn sym(id: u64, name: &str) -> Symbol { @@ -195,8 +195,70 @@ async fn seed_many(db: &str, count: usize) { idx.ingest(&results).await.unwrap(); } +/// Seed N symbol, mỗi symbol gắn annotation `@RestController` (chẵn) / `@Service` +/// (lẻ) — dùng test resumable cho `search_by_annotation`. +async fn seed_annotations(db: &str, count: usize) { + let mut idx = GraphIndex::open(db).await.unwrap(); + let mut results = Vec::new(); + for (id, i) in (SYMBOL_BASE..).zip(0..count) { + let mut s = sym(id, &format!("svc_{i}")); + s.annotations = vec![codegraph_core::Annotation { + name: (if i % 2 == 0 { "@RestController" } else { "@Service" }).into(), + args: HashMap::new(), + line: 1, + }]; + results.push(ParseResult { + path: "src/a.ts".into(), + language: "typescript".into(), + bytes: 10, + lines: 4, + symbols: vec![s], + chains: HashMap::new(), + calls: vec![], + }); + } + idx.ingest(&results).await.unwrap(); +} + +/// Seed N caller, mỗi caller gọi một library call lowercase `log.println` — dùng +/// test resumable cho `references` (call_name lowercase để khớp substring, do +/// engine filter `name.contains(&q)` với `q` đã lowercased). +async fn seed_references(db: &str, count: usize) { + let mut idx = GraphIndex::open(db).await.unwrap(); + let mut results = Vec::new(); + for i in 0..count { + let caller = SYMBOL_BASE + i as u64; + results.push(ParseResult { + path: "src/a.ts".into(), + language: "typescript".into(), + bytes: 10, + lines: 4, + symbols: vec![sym(caller, &format!("caller_{i}"))], + chains: HashMap::new(), + calls: vec![CallRecord { + caller_id: caller, + call_name: "log.println".to_string(), + position: 1, + arg_exprs: vec!["msg".into()], + line: 3, + condition: None, + is_loop_body: false, + effect: EffectType::Log, + effect_desc: None, + target_class: None, + target_method: None, + }], + }); + } + idx.ingest(&results).await.unwrap(); +} + /// Resume roundtrip: timeout → lấy resume id → retry cùng args + resume → kết /// quả đầy đủ, không lặp/không mất. Resume id sai → lỗi bảo retry không resume. +/// +/// Dùng [`TIMEOUT_EXPIRE_IMMEDIATELY`] để deadline **đã hết hạn ngay** → +/// `timed_out` được đảm bảo trên mọi máy (không phụ thuộc tốc độ đồng hồ tường +/// như `timeout_ms = 1`, vốn có thể "chạy quá nhanh" và skip luồng resume). #[tokio::test] async fn search_resumable_timeout_retry_roundtrip() { let dir = tempfile::tempdir().unwrap(); @@ -205,22 +267,17 @@ async fn search_resumable_timeout_retry_roundtrip() { seed_many(&db_str, 6000).await; let api = api(&db_str).await; - // Call 1: timeout_ms=1 — trên seed 6000 symbol debug build chắc chắn trễ - // hơn 1ms. Nếu máy quá nhanh (không timeout) test vẫn đúng — chỉ bỏ qua - // nhánh retry. total = 5000 vì name engine chặn cứng MAX_RESULTS tên distinct. + // Call 1: deadline đã hết hạn ngay → chắc chắn timed_out, sinh resume id. + // total = 5000 vì name engine chặn cứng MAX_RESULTS tên distinct. let capped = 5000; - let first = api.search_resumable("order", 20, None, 1).await.unwrap(); - let resume_id = if first.timed_out { - assert!(first.resume.is_some(), "timeout must carry a resume id"); - first.resume.unwrap() - } else { - // Hoàn tất ngay — verify kết quả rồi dừng (không cần retry). - let ids: std::collections::HashSet = first.page.iter().map(|s| s.id).collect(); - assert_eq!(ids.len(), first.page.len(), "no duplicate results"); - assert_eq!(first.total, capped); - assert_eq!(first.page.len(), 20); - return; - }; + let first = api + .search_resumable("order", 20, None, codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY) + .await + .unwrap(); + assert!(first.timed_out, "expired deadline must time out"); + let resume_id = first + .resume + .expect("timeout must carry a resume id"); // Retry: cùng args + resume, không giới hạn thời gian → hoàn tất. let out = api @@ -253,6 +310,188 @@ async fn search_resumable_timeout_retry_roundtrip() { ); } +/// Resumable timeout→retry cho `search_by_annotation` — deterministic qua +/// `TIMEOUT_EXPIRE_IMMEDIATELY` (deadline đã hết hạn ngay lập tức trên mọi máy). +#[tokio::test] +async fn annotation_search_resumable_timeout_retry() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("db.sqlite"); + let db_str = format!("sqlite://{}", db_path.to_string_lossy()); + seed_annotations(&db_str, 200).await; + let api = api(&db_str).await; + + // Lần 1: deadline hết hạn ngay → chắc chắn timed_out + mang resume id. + let first = api + .search_by_annotation_resumable( + "@RestController", + None, + Pagination { limit: 20, offset: 0 }, + None, + codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY, + ) + .await + .unwrap(); + assert!(first.timed_out, "expired deadline must time out"); + let resume_id = first.resume.expect("timeout must carry a resume id"); + + // Lần 2: retry cùng args + resume id, timeout_ms=0 → hoàn tất. + let out = api + .search_by_annotation_resumable( + "@RestController", + None, + Pagination { limit: 20, offset: 0 }, + Some(resume_id.clone()), + 0, + ) + .await + .unwrap(); + assert!(!out.timed_out); + // 200 symbol, i % 2 == 0 → 100 gắn @RestController. + assert_eq!(out.total, 100, "total must match full scan"); + assert_eq!(out.page.len(), 20); + assert!(out.resume.is_none()); + + // Resume id sai → lỗi. + assert!( + api.search_by_annotation_resumable( + "@RestController", + None, + Pagination { limit: 20, offset: 0 }, + Some("deadbeef00000000".into()), + 0, + ) + .await + .is_err(), + "unknown resume id must be rejected" + ); + // Resume id của query khác → lỗi. + assert!( + api.search_by_annotation_resumable( + "@Service", + None, + Pagination { limit: 20, offset: 0 }, + Some(resume_id), + 0, + ) + .await + .is_err(), + "resume id for a different query must be rejected" + ); +} + +/// Resumable timeout→retry cho `references` (deterministic). +#[tokio::test] +async fn references_resumable_timeout_retry() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("db.sqlite"); + let db_str = format!("sqlite://{}", db_path.to_string_lossy()); + seed_references(&db_str, 50).await; // 50 caller, mỗi gọi "log.println" + let api = api(&db_str).await; + + let first = api + .references_resumable( + "log.println", + Pagination { limit: 20, offset: 0 }, + None, + codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY, + ) + .await + .unwrap(); + assert!(first.timed_out, "expired deadline must time out"); + let resume_id = first.resume.expect("timeout must carry a resume id"); + + let out = api + .references_resumable( + "log.println", + Pagination { limit: 20, offset: 0 }, + Some(resume_id.clone()), + 0, + ) + .await + .unwrap(); + assert!(!out.timed_out); + assert_eq!(out.page.len(), 20); + let ids: HashSet = out.page.iter().map(|c| c.func_id).collect(); + assert_eq!(ids.len(), 20, "no duplicate callers across the page"); + assert!(out.resume.is_none()); + + // Resume id sai → lỗi. + assert!( + api.references_resumable( + "log.println", + Pagination { limit: 20, offset: 0 }, + Some("deadbeef00000000".into()), + 0, + ) + .await + .is_err(), + "unknown resume id must be rejected" + ); + // Resume id của query khác → lỗi. + assert!( + api.references_resumable( + "other.call", + Pagination { limit: 20, offset: 0 }, + Some(resume_id), + 0, + ) + .await + .is_err(), + "resume id for a different query must be rejected" + ); +} + +/// Resumable timeout→retry cho `search_flow_pattern` (deterministic). +#[tokio::test] +async fn flow_search_resumable_timeout_retry() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("db.sqlite"); + let db_str = format!("sqlite://{}", db_path.to_string_lossy()); + let (caller, callee, _helper) = seed_index(&db_str).await; + let api = api(&db_str).await; + + // Chain chứa callee → 2 hit (caller→[caller,callee], callee→[callee,helper]). + let pattern = callee.to_string(); + + let first = api + .search_flow_pattern_resumable( + &pattern, + Pagination { limit: 20, offset: 0 }, + None, + codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY, + ) + .await + .unwrap(); + assert!(first.timed_out, "expired deadline must time out"); + let resume_id = first.resume.expect("timeout must carry a resume id"); + + let out = api + .search_flow_pattern_resumable( + &pattern, + Pagination { limit: 20, offset: 0 }, + Some(resume_id.clone()), + 0, + ) + .await + .unwrap(); + assert!(!out.timed_out); + assert_eq!(out.page.len(), 2, "two functions have chain containing callee"); + assert!(out.resume.is_none()); + + // Resume id của pattern khác → lỗi. + assert!( + api.search_flow_pattern_resumable( + &caller.to_string(), + Pagination { limit: 20, offset: 0 }, + Some(resume_id), + 0, + ) + .await + .is_err(), + "resume id for a different pattern must be rejected" + ); +} + /// Phân trang qua resume (Paged cursor): call 1 limit=10 (timeout_ms=0) hoàn /// tất + còn page sau → resume id; call 2 cùng resume + offset=10 → page rời, /// tổng nhất quán. diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index e271bfa6e..d0dfb7021 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -253,6 +253,7 @@ impl GraphIndex { match Self::split_dsn(dsn) { Some(("sqlite", path)) => Self::open_sqlite_dispatch(path).await, Some(("lmdb", path)) => Self::open_lmdb_dispatch(path).await, + Some(("redis", _)) => Self::open_redis_dispatch(dsn).await, _ => Self::open_default(dsn).await, } } @@ -280,8 +281,19 @@ impl GraphIndex { Err(backend_unavailable("lmdb")) } + /// `redis://` rõ ràng — compile trường redis; không compile → báo lỗi. + #[cfg(feature = "redis")] + async fn open_redis_dispatch(dsn: &str) -> Result { + Self::open_redis(dsn).await + } + /// `redis://` rõ ràng nhưng feature không bật → không thể mở. + #[cfg(not(feature = "redis"))] + async fn open_redis_dispatch(_dsn: &str) -> Result { + Err(backend_unavailable("redis")) + } + /// Tách `scheme://` khỏi DSN: trả `(scheme, phần còn lại)` hoặc `None` - /// nếu không có scheme (plain path / redis url giữ nguyên). + /// nếu không có scheme (plain path). fn split_dsn(dsn: &str) -> Option<(&'static str, &str)> { if let Some(rest) = dsn.strip_prefix("sqlite://") { return Some(("sqlite", rest)); @@ -289,6 +301,9 @@ impl GraphIndex { if let Some(rest) = dsn.strip_prefix("lmdb://") { return Some(("lmdb", rest)); } + if dsn.starts_with("redis://") || dsn.starts_with("rediss://") { + return Some(("redis", dsn)); + } None } @@ -308,11 +323,6 @@ impl GraphIndex { { return Self::open_lmdb(dsn).await; } - // Chỉ redis được compile — plain path = redis. - #[cfg(all(feature = "redis", not(any(feature = "sqlite", feature = "lmdb"))))] - { - return Self::open_redis(dsn).await; - } // Nhiều backend (≥2) — DSN không nói scheme → mơ hồ. #[cfg(any( all(feature = "sqlite", feature = "lmdb"), @@ -1514,6 +1524,48 @@ impl GraphIndex { Ok(out) } + /// Resumable + deadline-aware của [`Self::search_flow`]. `deadline` hết hạn + /// giữa chừng → trả `(Vec::new(), Some(offset))` (không kết quả nửa chừng), + /// caller retry với `offset` tiếp tục. `deadline = None` = chạy tới cùng. + pub async fn search_flow_resumable( + &self, + pattern: &[u64], + offset: usize, + limit: usize, + deadline: Option, + ) -> Result<(Vec, Option)> { + if pattern.is_empty() { + return Ok((Vec::new(), None)); + } + let hits = match self.chains.search(pattern, None).await { + Ok(h) => h, + Err(_) => return Ok((Vec::new(), None)), + }; + let limit = if limit == 0 { usize::MAX } else { limit }; + let cap = if limit == usize::MAX { usize::MAX } else { offset + limit }; + let mut out = Vec::new(); + for (record, _) in hits { + if deadline.is_some_and(|dl| Instant::now() >= dl) { + return Ok((Vec::new(), Some(offset))); + } + let func_id = record as u64; + if let Some(sym) = self.symbols.get(&func_id) { + let chain = self.chains_map.get(&func_id).cloned().unwrap_or_default(); + out.push(SearchFlowResult { + function_id: func_id, + function_name: sym.name.clone(), + chain, + match_count: 1, + }); + } + if out.len() >= cap { + break; + } + } + let page = out.into_iter().skip(offset).take(limit).collect::>(); + Ok((page, None)) + } + /// Tìm function gọi một library call có tên chứa `query` (case-insensitive /// substring trên call-name index, kể cả call unresolved). Gom theo caller, /// sort theo FuncName rồi FuncID. @@ -1556,6 +1608,59 @@ impl GraphIndex { Ok(out) } + /// Resumable + deadline-aware của [`Self::callers_by_call_name`]. Tương tự + /// [`Self::search_flow_resumable`]: timeout → `(Vec::new(), Some(offset))`, + /// caller retry với `offset` tiếp tục. `deadline = None` = chạy tới cùng. + pub async fn callers_by_call_name_resumable( + &self, + query: &str, + offset: usize, + limit: usize, + deadline: Option, + ) -> Result<(Vec, Option)> { + let q = query.to_lowercase(); + let mut matched: Vec<(&String, &Vec)> = self + .call_names + .iter() + .filter(|(name, _)| name.contains(&q)) + .collect(); + if deadline.is_some_and(|dl| Instant::now() >= dl) { + return Ok((Vec::new(), Some(offset))); + } + matched.sort_by_key(|(name, _)| (*name).clone()); + let mut by_func: HashMap = HashMap::new(); + for (_, sites) in matched { + if deadline.is_some_and(|dl| Instant::now() >= dl) { + return Ok((Vec::new(), Some(offset))); + } + for site in sites { + let entry = by_func.entry(site.caller_id).or_insert_with(|| { + let sym = self.symbols.get(&site.caller_id); + CallSiteResult { + func_id: site.caller_id, + func_name: sym.map(|s| s.name.clone()).unwrap_or_default(), + file: sym.map(|s| s.file.clone()).unwrap_or_default(), + call_sites: Vec::new(), + } + }); + entry.call_sites.push(site.clone()); + } + } + let mut out: Vec = by_func.into_values().collect(); + out.sort_by(|a, b| { + a.func_name + .cmp(&b.func_name) + .then(a.func_id.cmp(&b.func_id)) + }); + let limit = if limit == 0 { usize::MAX } else { limit }; + let cap = if limit == usize::MAX { usize::MAX } else { offset + limit }; + if out.len() > cap { + out.truncate(cap); + } + let page = out.into_iter().skip(offset).take(limit).collect::>(); + Ok((page, None)) + } + /// Files trong graph. pub fn files(&self) -> Vec { self.files.clone() @@ -1663,6 +1768,32 @@ impl GraphIndex { (all.into_iter().skip(offset).take(limit).collect(), total) } + /// Resumable + deadline-aware của [`Self::list_symbols_by_kind`]. Timeout → + /// `(Vec::new(), 0, Some(offset))` (không kết quả nửa chừng), retry với + /// `offset` tiếp tục. `deadline = None` = chạy tới cùng. + pub fn list_symbols_by_kind_resumable( + &self, + kind: SymbolKind, + offset: usize, + limit: usize, + deadline: Option, + ) -> (Vec, usize, Option) { + let mut all: Vec = Vec::new(); + for s in self.symbols.values() { + if deadline.is_some_and(|dl| Instant::now() >= dl) { + return (Vec::new(), 0, Some(offset)); + } + if s.kind == kind { + all.push(s.clone()); + } + } + all.sort_by(|a, b| a.name.cmp(&b.name).then(a.id.cmp(&b.id))); + let total = all.len(); + let limit = if limit == 0 { usize::MAX } else { limit }; + let page = all.into_iter().skip(offset).take(limit).collect::>(); + (page, total, None) + } + /// Tìm symbol theo annotation (case-insensitive substring trên tên /// annotation). Trả về (page, total, truncated) — total là con số thật, /// truncated=true khi còn trang sau. @@ -1693,6 +1824,38 @@ impl GraphIndex { (page, total, truncated) } + /// Resumable + deadline-aware của [`Self::search_by_annotation`]. Timeout → + /// `(Vec::new(), 0, Some(offset))` (không kết quả nửa chừng), retry với + /// `offset` tiếp tục. `deadline = None` = chạy tới cùng. + pub fn search_by_annotation_resumable( + &self, + annotation: &str, + kind: Option, + offset: usize, + limit: usize, + deadline: Option, + ) -> (Vec, usize, Option) { + let q = annotation.to_lowercase(); + let mut all: Vec = Vec::new(); + for s in self.symbols.values() { + if deadline.is_some_and(|dl| Instant::now() >= dl) { + return (Vec::new(), 0, Some(offset)); + } + if s.annotations + .iter() + .any(|a| a.name.to_lowercase().contains(&q)) + && kind.is_none_or(|k| s.kind == k) + { + all.push(s.clone()); + } + } + all.sort_by(|a, b| a.name.cmp(&b.name).then(a.id.cmp(&b.id))); + let total = all.len(); + let limit = if limit == 0 { usize::MAX } else { limit }; + let page = all.into_iter().skip(offset).take(limit).collect::>(); + (page, total, None) + } + /// Ước lượng dependencies từ call names: tách module prefix (phần trước dấu /// chấm đầu tiên) — internal nếu có symbol trong repo mang chính tên đó, /// external còn lại. Sort theo số call sites giảm dần. diff --git a/crates/codegraph-mcp/src/server-instructions.md b/crates/codegraph-mcp/src/server-instructions.md index 783c9a298..5c6940367 100644 --- a/crates/codegraph-mcp/src/server-instructions.md +++ b/crates/codegraph-mcp/src/server-instructions.md @@ -76,14 +76,16 @@ finds every `*Service` class), and `exact`. Use `total` + `offset` to page. ## Large indexes: timeout + resume -On very large indexes a broad search (`codegraph_search` / `codegraph_search_symbol`) -can exceed its time budget. Both tools accept `timeout_ms` (default `2000`; +On very large indexes a broad search (`codegraph_search`, `codegraph_search_symbol`, +`codegraph_search_by_annotation`, `codegraph_search_flow`, `codegraph_references`, +`codegraph_search_by_call`, `codegraph_list_classes`, `codegraph_list_interfaces`) +can exceed its time budget. All of these tools accept `timeout_ms` (default `20000`; `0` = no limit). When the budget runs out mid-search the tool **errors** and does NOT return partial results — the message includes `"resume": ""` and a progress count: ``` -codegraph_search_symbol timed out after 2000ms (collected 134 symbols so far). +codegraph_search_symbol timed out after 20000ms (collected 134 symbols so far). Retry the same call with the same arguments plus "resume": "" to continue the search from where it stopped. ``` diff --git a/crates/codegraph-mcp/src/tools.rs b/crates/codegraph-mcp/src/tools.rs index d0430987d..03fef1dc1 100644 --- a/crates/codegraph-mcp/src/tools.rs +++ b/crates/codegraph-mcp/src/tools.rs @@ -40,12 +40,12 @@ fn tool_defs() -> Vec { vec![ tool( "codegraph_search", - "Search symbols by name (substring, case-insensitive). On large indexes this can take a while — pass timeout_ms (default 2000) and, if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue the search from where it stopped.", + "Search symbols by name (substring, case-insensitive). On large indexes this can take a while — pass timeout_ms (default 20000) and, if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue the search from where it stopped.", json!({ "type": "object", "properties": { "query": { "type": "string" }, "limit": { "type": "integer", "default": 10 }, "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." }, - "timeout_ms": { "type": "integer", "default": 2000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, "detail": { "type": "string", "enum": ["minimal", "medium", "verbose"], "description": "Symbol detail for this call (overrides session default): minimal = id/name/kind/file/line, medium = + signature, verbose = full Symbol." }, "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." } }, "required": ["query"] }), @@ -99,9 +99,13 @@ fn tool_defs() -> Vec { ), tool( "codegraph_search_flow", - "Find functions whose call chain contains a pattern. Pattern = comma-separated tokens: numeric ids, marker names (LOOP, IF_TRUE, IF_FALSE, BRANCH_END, RETURN, LOOP_BACK, SWITCH_CASE, SWITCH_END, BREAK, CONTINUE, THROW) or symbol names.", + "Find functions whose call chain contains a pattern. Pattern = comma-separated tokens: numeric ids, marker names (LOOP, IF_TRUE, IF_FALSE, BRANCH_END, RETURN, LOOP_BACK, SWITCH_CASE, SWITCH_END, BREAK, CONTINUE, THROW) or symbol names. On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue.", json!({ "type": "object", "properties": { - "pattern": { "type": "string" } + "pattern": { "type": "string" }, + "limit": { "type": "integer", "default": 20 }, + "offset": { "type": "integer", "default": 0 }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." } }, "required": ["pattern"] }), ), tool( @@ -116,10 +120,13 @@ fn tool_defs() -> Vec { ), tool( "codegraph_references", - "Functions that call a library call whose name contains the query (includes unresolved external calls).", + "Functions that call a library call whose name contains the query (includes unresolved external calls). On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue.", json!({ "type": "object", "properties": { "query": { "type": "string" }, - "limit": { "type": "integer", "default": 10 } + "limit": { "type": "integer", "default": 10 }, + "offset": { "type": "integer", "default": 0 }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." } }, "required": ["query"] }), ), tool( @@ -156,7 +163,7 @@ fn tool_defs() -> Vec { // ── Enhanced symbol search (semgraph_search_symbol) ── tool( "codegraph_search_symbol", - "Search symbols by name with optional kind filter, match mode, and pagination. match: 'contains' (substring anywhere, default), 'prefix' (name starts with), 'suffix' (name ENDS with — e.g. query=\"Service\" finds every *Service class), 'exact' (exact name, case-insensitive). Use 'total' with 'offset' to fetch further pages until offset >= total. On large indexes pass timeout_ms (default 2000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue. When more results remain, the response includes a resume id you can pass to page further without re-scanning.", + "Search symbols by name with optional kind filter, match mode, and pagination. match: 'contains' (substring anywhere, default), 'prefix' (name starts with), 'suffix' (name ENDS with — e.g. query=\"Service\" finds every *Service class), 'exact' (exact name, case-insensitive). Use 'total' with 'offset' to fetch further pages until offset >= total. On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue. When more results remain, the response includes a resume id you can pass to page further without re-scanning.", json!({ "type": "object", "properties": { "query": { "type": "string" }, "kind": { "type": "string", "enum": ["function", "method", "class", "interface", "enum", "variable", "constant", "parameter", "field", "module", "file"] }, @@ -164,7 +171,7 @@ fn tool_defs() -> Vec { "limit": { "type": "integer", "default": 20 }, "offset": { "type": "integer", "default": 0 }, "resume": { "type": "string", "description": "Resume id from a previous timeout (or from a previous response with more pages) — retry the same call with this to continue where it stopped." }, - "timeout_ms": { "type": "integer", "default": 2000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, "detail": { "type": "string", "enum": ["minimal", "medium", "verbose"], "description": "Symbol detail for this call (overrides session default): minimal = id/name/kind/file/line, medium = + signature, verbose = full Symbol." }, "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." } }, "required": ["query"] }), @@ -191,22 +198,26 @@ fn tool_defs() -> Vec { ), tool( "codegraph_list_classes", - "List all class symbols in the index (paginated).", + "List all class symbols in the index (paginated). On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue.", json!({ "type": "object", "properties": { "limit": { "type": "integer", "default": 20 }, "offset": { "type": "integer", "default": 0 }, "detail": { "type": "string", "enum": ["minimal", "medium", "verbose"], "description": "Symbol detail for this call (overrides session default): minimal = id/name/kind/file/line, medium = + signature, verbose = full Symbol." }, - "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." } + "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." } } }), ), tool( "codegraph_list_interfaces", - "List all interface symbols in the index (paginated).", + "List all interface symbols in the index (paginated). On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue.", json!({ "type": "object", "properties": { "limit": { "type": "integer", "default": 20 }, "offset": { "type": "integer", "default": 0 }, "detail": { "type": "string", "enum": ["minimal", "medium", "verbose"], "description": "Symbol detail for this call (overrides session default): minimal = id/name/kind/file/line, medium = + signature, verbose = full Symbol." }, - "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." } + "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." } } }), ), tool( @@ -221,22 +232,27 @@ fn tool_defs() -> Vec { // ── Annotation / call / dependency queries ── tool( "codegraph_search_by_annotation", - "Search symbols by annotation (e.g. @RestController, @GetMapping, @Autowired, @Override). Case-insensitive substring match. Optional kind filter.", + "Search symbols by annotation (e.g. @RestController, @GetMapping, @Autowired, @Override). Case-insensitive substring match. Optional kind filter. On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue.", json!({ "type": "object", "properties": { "annotation": { "type": "string" }, "kind": { "type": "string", "enum": ["function", "method", "class", "interface", "enum", "variable", "constant", "parameter", "field", "module", "file"] }, "limit": { "type": "integer", "default": 20 }, "offset": { "type": "integer", "default": 0 }, "detail": { "type": "string", "enum": ["minimal", "medium", "verbose"], "description": "Symbol detail for this call (overrides session default): minimal = id/name/kind/file/line, medium = + signature, verbose = full Symbol." }, - "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." } + "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." } }, "required": ["annotation"] }), ), tool( "codegraph_search_by_call", - "Find functions that call a given class/method name inside their bodies (e.g. \"LogManager\" or \"LogManager.getLogger\"). Matches ALL call names captured by the parser — including external library calls that don't resolve to in-repo symbols. Each result includes per-call-site context: line, surrounding condition, whether inside a loop, and the call arguments.", + "Find functions that call a given class/method name inside their bodies (e.g. \"LogManager\" or \"LogManager.getLogger\"). Matches ALL call names captured by the parser — including external library calls that don't resolve to in-repo symbols. Each result includes per-call-site context: line, surrounding condition, whether inside a loop, and the call arguments. On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue.", json!({ "type": "object", "properties": { "call_name": { "type": "string" }, - "limit": { "type": "integer", "default": 20 } + "limit": { "type": "integer", "default": 20 }, + "offset": { "type": "integer", "default": 0 }, + "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, + "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." } }, "required": ["call_name"] }), ), tool( @@ -320,7 +336,7 @@ pub async fn dispatch_with_api( let timeout_ms = args .get("timeout_ms") .and_then(|v| v.as_u64()) - .unwrap_or(2000); + .unwrap_or(20000); let out = api.search_resumable(q, limit, resume, timeout_ms).await?; if out.timed_out { // Không trả kết quả nửa chừng — báo lỗi kèm resume id để LLM retry @@ -433,8 +449,30 @@ pub async fn dispatch_with_api( } "codegraph_search_flow" => { let pattern = arg_str(&args, "pattern")?; - let hits = api.search_flow_pattern(pattern).await?; - emit(root.as_str(), &hits) + let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(20) as u32; + let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + let resume = args + .get("resume") + .and_then(|v| v.as_str()) + .map(str::to_string); + let timeout_ms = args + .get("timeout_ms") + .and_then(|v| v.as_u64()) + .unwrap_or(20000); + let out = api + .search_flow_pattern_resumable(pattern, Pagination { limit, offset }, resume, timeout_ms) + .await?; + if out.timed_out { + return Err(Error::Other(format!( + "codegraph_search_flow timed out after {}ms (collected {} results so far). \ + Retry the same call with the same arguments plus \"resume\": \"{}\" \ + to continue the search from where it stopped.", + timeout_ms, + out.progress, + out.resume.as_deref().unwrap_or("") + ))); + } + emit(root.as_str(), &out.page) } "codegraph_context" => { let req = ContextRequest { @@ -453,8 +491,29 @@ pub async fn dispatch_with_api( "codegraph_references" => { let q = arg_str(&args, "query")?; let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(10) as u32; - let report = api.references(q, limit).await?; - emit(root.as_str(), &report) + let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + let resume = args + .get("resume") + .and_then(|v| v.as_str()) + .map(str::to_string); + let timeout_ms = args + .get("timeout_ms") + .and_then(|v| v.as_u64()) + .unwrap_or(20000); + let out = api + .references_resumable(q, Pagination { limit, offset }, resume, timeout_ms) + .await?; + if out.timed_out { + return Err(Error::Other(format!( + "codegraph_references timed out after {}ms (collected {} results so far). \ + Retry the same call with the same arguments plus \"resume\": \"{}\" \ + to continue the search from where it stopped.", + timeout_ms, + out.progress, + out.resume.as_deref().unwrap_or("") + ))); + } + emit(root.as_str(), &out.page) } "codegraph_files" => { let prefix = args.get("path").and_then(|v| v.as_str()).unwrap_or(""); @@ -498,7 +557,7 @@ pub async fn dispatch_with_api( let timeout_ms = args .get("timeout_ms") .and_then(|v| v.as_u64()) - .unwrap_or(2000); + .unwrap_or(20000); let out = api .search_symbol_paged_resumable( q, @@ -623,10 +682,31 @@ pub async fn dispatch_with_api( "codegraph_list_classes" => { let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(20) as u32; let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let (results, total) = api.list_by_kind(SymbolKind::Class, limit, offset).await; + let resume = args + .get("resume") + .and_then(|v| v.as_str()) + .map(str::to_string); + let timeout_ms = args + .get("timeout_ms") + .and_then(|v| v.as_u64()) + .unwrap_or(20000); + let out = api + .list_by_kind_resumable(SymbolKind::Class, Pagination { limit, offset }, resume, timeout_ms) + .await?; + if out.timed_out { + return Err(Error::Other(format!( + "codegraph_list_classes timed out after {}ms (collected {} symbols so far). \ + Retry the same call with the same arguments plus \"resume\": \"{}\" \ + to continue from where it stopped.", + timeout_ms, + out.progress, + out.resume.as_deref().unwrap_or("") + ))); + } let detail = detail_from_args(&args, session_detail); let format = format_from_args(&args, session_format); - let results: Vec = results + let results: Vec = out + .page .into_iter() .map(|s| symbol_json(root.as_str(), &s, detail, format)) .collect(); @@ -635,19 +715,41 @@ pub async fn dispatch_with_api( json!({ "kind": "class", "results": results, - "total": total, + "total": out.total, "limit": limit, "offset": offset, + "resume": out.resume, }), ) } "codegraph_list_interfaces" => { let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(20) as u32; let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let (results, total) = api.list_by_kind(SymbolKind::Interface, limit, offset).await; + let resume = args + .get("resume") + .and_then(|v| v.as_str()) + .map(str::to_string); + let timeout_ms = args + .get("timeout_ms") + .and_then(|v| v.as_u64()) + .unwrap_or(20000); + let out = api + .list_by_kind_resumable(SymbolKind::Interface, Pagination { limit, offset }, resume, timeout_ms) + .await?; + if out.timed_out { + return Err(Error::Other(format!( + "codegraph_list_interfaces timed out after {}ms (collected {} symbols so far). \ + Retry the same call with the same arguments plus \"resume\": \"{}\" \ + to continue from where it stopped.", + timeout_ms, + out.progress, + out.resume.as_deref().unwrap_or("") + ))); + } let detail = detail_from_args(&args, session_detail); let format = format_from_args(&args, session_format); - let results: Vec = results + let results: Vec = out + .page .into_iter() .map(|s| symbol_json(root.as_str(), &s, detail, format)) .collect(); @@ -656,9 +758,10 @@ pub async fn dispatch_with_api( json!({ "kind": "interface", "results": results, - "total": total, + "total": out.total, "limit": limit, "offset": offset, + "resume": out.resume, }), ) } @@ -709,12 +812,31 @@ pub async fn dispatch_with_api( .and_then(SymbolKind::parse); let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(20) as u32; let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let (results, total, truncated) = api - .search_by_annotation(annotation, kind, offset, limit) - .await; + let resume = args + .get("resume") + .and_then(|v| v.as_str()) + .map(str::to_string); + let timeout_ms = args + .get("timeout_ms") + .and_then(|v| v.as_u64()) + .unwrap_or(20000); + let out = api + .search_by_annotation_resumable(annotation, kind, Pagination { limit, offset }, resume, timeout_ms) + .await?; + if out.timed_out { + return Err(Error::Other(format!( + "codegraph_search_by_annotation timed out after {}ms (collected {} symbols so far). \ + Retry the same call with the same arguments plus \"resume\": \"{}\" \ + to continue the search from where it stopped.", + timeout_ms, + out.progress, + out.resume.as_deref().unwrap_or("") + ))); + } let detail = detail_from_args(&args, session_detail); let format = format_from_args(&args, session_format); - let results: Vec = results + let results: Vec = out + .page .into_iter() .map(|s| symbol_json(root.as_str(), &s, detail, format)) .collect(); @@ -724,22 +846,44 @@ pub async fn dispatch_with_api( "annotation": annotation, "kind": kind.map(|k| k.as_str()), "results": results, - "total": total, + "total": out.total, "offset": offset, - "truncated": truncated, + "resume": out.resume, }), ) } "codegraph_search_by_call" => { let call_name = arg_str(&args, "call_name")?; let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(20) as u32; - let hits = api.references(call_name, limit).await?; + let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + let resume = args + .get("resume") + .and_then(|v| v.as_str()) + .map(str::to_string); + let timeout_ms = args + .get("timeout_ms") + .and_then(|v| v.as_u64()) + .unwrap_or(20000); + let out = api + .references_resumable(call_name, Pagination { limit, offset }, resume, timeout_ms) + .await?; + if out.timed_out { + return Err(Error::Other(format!( + "codegraph_search_by_call timed out after {}ms (collected {} results so far). \ + Retry the same call with the same arguments plus \"resume\": \"{}\" \ + to continue the search from where it stopped.", + timeout_ms, + out.progress, + out.resume.as_deref().unwrap_or("") + ))); + } emit_value( root.as_str(), json!({ "call_name": call_name, - "results": hits, - "total": hits.len(), + "results": out.page, + "total": out.page.len(), + "resume": out.resume, }), ) } From 2584dd5b5c711a706336ce1902aa7c545aac57c9 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 15 Aug 2026 20:25:54 +0700 Subject: [PATCH 6/9] Fix tests --- crates/codegraph-api/src/lib.rs | 13 ++-- crates/codegraph-api/tests/api.rs | 72 ++++++++++++++++----- crates/codegraph-graph/src/lib.rs | 12 +++- crates/codegraph-graph/src/storage/mysql.rs | 6 +- crates/codegraph-mcp/src/tools.rs | 29 +++++++-- sql/mysql/002-add-repos-registry.sql | 2 +- sql/postgres/002-add-repos-registry.sql | 2 +- 7 files changed, 102 insertions(+), 34 deletions(-) diff --git a/crates/codegraph-api/src/lib.rs b/crates/codegraph-api/src/lib.rs index 641044604..551bedf7a 100644 --- a/crates/codegraph-api/src/lib.rs +++ b/crates/codegraph-api/src/lib.rs @@ -298,15 +298,14 @@ impl GraphApi { .into(), )); } - let c = match stored { - ResumeCursor::Name(c) => c, - _ => { - return Err(Error::Invalid( + let c = + match stored { + ResumeCursor::Name(c) => c, + _ => return Err(Error::Invalid( "resume id was created for a different query — retry without resume" .into(), - )) - } - }; + )), + }; if c.query != q || c.mode != mode || c.kind != kind { return Err(Error::Invalid( "resume id was created for a different query — retry without resume".into(), diff --git a/crates/codegraph-api/tests/api.rs b/crates/codegraph-api/tests/api.rs index 1c8e8cbc3..d75595524 100644 --- a/crates/codegraph-api/tests/api.rs +++ b/crates/codegraph-api/tests/api.rs @@ -203,7 +203,12 @@ async fn seed_annotations(db: &str, count: usize) { for (id, i) in (SYMBOL_BASE..).zip(0..count) { let mut s = sym(id, &format!("svc_{i}")); s.annotations = vec![codegraph_core::Annotation { - name: (if i % 2 == 0 { "@RestController" } else { "@Service" }).into(), + name: (if i % 2 == 0 { + "@RestController" + } else { + "@Service" + }) + .into(), args: HashMap::new(), line: 1, }]; @@ -275,9 +280,7 @@ async fn search_resumable_timeout_retry_roundtrip() { .await .unwrap(); assert!(first.timed_out, "expired deadline must time out"); - let resume_id = first - .resume - .expect("timeout must carry a resume id"); + let resume_id = first.resume.expect("timeout must carry a resume id"); // Retry: cùng args + resume, không giới hạn thời gian → hoàn tất. let out = api @@ -325,7 +328,10 @@ async fn annotation_search_resumable_timeout_retry() { .search_by_annotation_resumable( "@RestController", None, - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0, + }, None, codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY, ) @@ -339,7 +345,10 @@ async fn annotation_search_resumable_timeout_retry() { .search_by_annotation_resumable( "@RestController", None, - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0, + }, Some(resume_id.clone()), 0, ) @@ -356,7 +365,10 @@ async fn annotation_search_resumable_timeout_retry() { api.search_by_annotation_resumable( "@RestController", None, - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0 + }, Some("deadbeef00000000".into()), 0, ) @@ -369,7 +381,10 @@ async fn annotation_search_resumable_timeout_retry() { api.search_by_annotation_resumable( "@Service", None, - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0 + }, Some(resume_id), 0, ) @@ -391,7 +406,10 @@ async fn references_resumable_timeout_retry() { let first = api .references_resumable( "log.println", - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0, + }, None, codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY, ) @@ -403,7 +421,10 @@ async fn references_resumable_timeout_retry() { let out = api .references_resumable( "log.println", - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0, + }, Some(resume_id.clone()), 0, ) @@ -419,7 +440,10 @@ async fn references_resumable_timeout_retry() { assert!( api.references_resumable( "log.println", - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0 + }, Some("deadbeef00000000".into()), 0, ) @@ -431,7 +455,10 @@ async fn references_resumable_timeout_retry() { assert!( api.references_resumable( "other.call", - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0 + }, Some(resume_id), 0, ) @@ -456,7 +483,10 @@ async fn flow_search_resumable_timeout_retry() { let first = api .search_flow_pattern_resumable( &pattern, - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0, + }, None, codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY, ) @@ -468,21 +498,31 @@ async fn flow_search_resumable_timeout_retry() { let out = api .search_flow_pattern_resumable( &pattern, - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0, + }, Some(resume_id.clone()), 0, ) .await .unwrap(); assert!(!out.timed_out); - assert_eq!(out.page.len(), 2, "two functions have chain containing callee"); + assert_eq!( + out.page.len(), + 2, + "two functions have chain containing callee" + ); assert!(out.resume.is_none()); // Resume id của pattern khác → lỗi. assert!( api.search_flow_pattern_resumable( &caller.to_string(), - Pagination { limit: 20, offset: 0 }, + Pagination { + limit: 20, + offset: 0 + }, Some(resume_id), 0, ) diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index d0dfb7021..6880902c9 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -1542,7 +1542,11 @@ impl GraphIndex { Err(_) => return Ok((Vec::new(), None)), }; let limit = if limit == 0 { usize::MAX } else { limit }; - let cap = if limit == usize::MAX { usize::MAX } else { offset + limit }; + let cap = if limit == usize::MAX { + usize::MAX + } else { + offset + limit + }; let mut out = Vec::new(); for (record, _) in hits { if deadline.is_some_and(|dl| Instant::now() >= dl) { @@ -1653,7 +1657,11 @@ impl GraphIndex { .then(a.func_id.cmp(&b.func_id)) }); let limit = if limit == 0 { usize::MAX } else { limit }; - let cap = if limit == usize::MAX { usize::MAX } else { offset + limit }; + let cap = if limit == usize::MAX { + usize::MAX + } else { + offset + limit + }; if out.len() > cap { out.truncate(cap); } diff --git a/crates/codegraph-graph/src/storage/mysql.rs b/crates/codegraph-graph/src/storage/mysql.rs index 2ca22f3db..e5b3d40a7 100644 --- a/crates/codegraph-graph/src/storage/mysql.rs +++ b/crates/codegraph-graph/src/storage/mysql.rs @@ -588,9 +588,9 @@ impl Storage for MySqlStorage { async fn upsert_file(&mut self, f: &FileInfo) -> Result<()> { sqlx::query( - "INSERT INTO sg_files (repo_id, path, language, bytes, lines) VALUES (?, ?, ?, ?, ?) \ + "INSERT INTO sg_files (repo_id, path, language, bytes, `lines`) VALUES (?, ?, ?, ?, ?) \ ON DUPLICATE KEY UPDATE \ - language = VALUES(language), bytes = VALUES(bytes), lines = VALUES(lines)", + language = VALUES(language), bytes = VALUES(bytes), `lines` = VALUES(`lines`)", ) .bind(self.repo_id as i64) .bind(&f.path) @@ -605,7 +605,7 @@ impl Storage for MySqlStorage { async fn load_all_files(&self) -> Result> { let rows = - sqlx::query("SELECT path, language, bytes, lines FROM sg_files WHERE repo_id = ?") + sqlx::query("SELECT path, language, bytes, `lines` FROM sg_files WHERE repo_id = ?") .bind(self.repo_id as i64) .fetch_all(&self.pool) .await diff --git a/crates/codegraph-mcp/src/tools.rs b/crates/codegraph-mcp/src/tools.rs index 03fef1dc1..65bebe0ec 100644 --- a/crates/codegraph-mcp/src/tools.rs +++ b/crates/codegraph-mcp/src/tools.rs @@ -460,7 +460,12 @@ pub async fn dispatch_with_api( .and_then(|v| v.as_u64()) .unwrap_or(20000); let out = api - .search_flow_pattern_resumable(pattern, Pagination { limit, offset }, resume, timeout_ms) + .search_flow_pattern_resumable( + pattern, + Pagination { limit, offset }, + resume, + timeout_ms, + ) .await?; if out.timed_out { return Err(Error::Other(format!( @@ -691,7 +696,12 @@ pub async fn dispatch_with_api( .and_then(|v| v.as_u64()) .unwrap_or(20000); let out = api - .list_by_kind_resumable(SymbolKind::Class, Pagination { limit, offset }, resume, timeout_ms) + .list_by_kind_resumable( + SymbolKind::Class, + Pagination { limit, offset }, + resume, + timeout_ms, + ) .await?; if out.timed_out { return Err(Error::Other(format!( @@ -734,7 +744,12 @@ pub async fn dispatch_with_api( .and_then(|v| v.as_u64()) .unwrap_or(20000); let out = api - .list_by_kind_resumable(SymbolKind::Interface, Pagination { limit, offset }, resume, timeout_ms) + .list_by_kind_resumable( + SymbolKind::Interface, + Pagination { limit, offset }, + resume, + timeout_ms, + ) .await?; if out.timed_out { return Err(Error::Other(format!( @@ -821,7 +836,13 @@ pub async fn dispatch_with_api( .and_then(|v| v.as_u64()) .unwrap_or(20000); let out = api - .search_by_annotation_resumable(annotation, kind, Pagination { limit, offset }, resume, timeout_ms) + .search_by_annotation_resumable( + annotation, + kind, + Pagination { limit, offset }, + resume, + timeout_ms, + ) .await?; if out.timed_out { return Err(Error::Other(format!( diff --git a/sql/mysql/002-add-repos-registry.sql b/sql/mysql/002-add-repos-registry.sql index fc12a464f..c2d4b220b 100644 --- a/sql/mysql/002-add-repos-registry.sql +++ b/sql/mysql/002-add-repos-registry.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS repos ( repo_id BIGINT NOT NULL PRIMARY KEY, -- repo_id (số u64, random lúc init) shard INT NOT NULL, -- shard server được gán (index vào dsns) - root VARCHAR(700) NOT NULL, -- root path chuẩn để lookup + root VARCHAR(700), -- root path chuẩn để lookup (nullable: route có thể không có root) created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- Lookup theo root (adopt cùng repo_id cho clone/máy khác). diff --git a/sql/postgres/002-add-repos-registry.sql b/sql/postgres/002-add-repos-registry.sql index 4d1de7840..f3da36a44 100644 --- a/sql/postgres/002-add-repos-registry.sql +++ b/sql/postgres/002-add-repos-registry.sql @@ -30,7 +30,7 @@ BEGIN; CREATE TABLE IF NOT EXISTS repos ( repo_id BIGINT NOT NULL PRIMARY KEY, -- repo_id (số u64, random lúc init) shard INT NOT NULL, -- shard server được gán (index vào dsns) - root TEXT NOT NULL, -- root path chuẩn để lookup + root TEXT, -- root path chuẩn để lookup (nullable: route có thể không có root) created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); -- Lookup theo root (adopt cùng repo_id cho clone/máy khác). From 420efae951a39dc291ad75e8a48e584825607232 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 15 Aug 2026 21:05:05 +0700 Subject: [PATCH 7/9] Fix tests --- crates/codegraph-graph/tests/rdbms.rs | 6 ++-- sql/README.md | 2 +- sql/mysql/001-initial-schema.sql | 44 ++++++++++++++------------- sql/mysql/002-add-repos-registry.sql | 2 +- sql/postgres/001-initial-schema.sql | 2 +- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/crates/codegraph-graph/tests/rdbms.rs b/crates/codegraph-graph/tests/rdbms.rs index 5010b1ede..c6b60d813 100644 --- a/crates/codegraph-graph/tests/rdbms.rs +++ b/crates/codegraph-graph/tests/rdbms.rs @@ -72,7 +72,8 @@ async fn rdbms_ingest_reopen_roundtrip() { let repo_id: u64 = std::env::var("TEST_RDBMS_REPO_ID") .ok() .and_then(|s| s.parse().ok()) - .unwrap_or(0); + .unwrap_or(0) + + 100; // partition riêng: test chạy song song với rdbms_empty_ingest_wipes_store let route = StorageRoute::Sharded { dsns: vec![dsn], @@ -138,7 +139,8 @@ async fn rdbms_empty_ingest_wipes_store() { let repo_id: u64 = std::env::var("TEST_RDBMS_REPO_ID") .ok() .and_then(|s| s.parse().ok()) - .unwrap_or(0); + .unwrap_or(0) + + 200; // partition riêng: test chạy song song với rdbms_ingest_reopen_roundtrip let route = StorageRoute::Sharded { dsns: vec![dsn], diff --git a/sql/README.md b/sql/README.md index 660503d45..41ff83f0c 100644 --- a/sql/README.md +++ b/sql/README.md @@ -64,7 +64,7 @@ parquet ở phase DuckDB): | Bảng | PK | Nội dung | |---|---|---| -| `sg_symbols` | `(repo_id, id)` | `Symbol` — cột thật; `annotations` là cột JSON | +| `sg_symbols` | `(repo_id, id)` | `Symbol` — cột thật; `annotations` là cột `TEXT` (app lưu JSON string qua `serde_json`, đọc bằng `from_str` — không dùng JSON/JSONB để sqlx decode `String` được) | | `sg_files` | `(repo_id, path)` | `FileInfo` | | `sg_call_records` | `(repo_id, func)` | call records của từng function (JSON bytes) | | `sg_call_names` | `(repo_id, name)` | inverted index call name → call sites (JSON bytes) | diff --git a/sql/mysql/001-initial-schema.sql b/sql/mysql/001-initial-schema.sql index 3edbe651d..f5eef88db 100644 --- a/sql/mysql/001-initial-schema.sql +++ b/sql/mysql/001-initial-schema.sql @@ -10,8 +10,10 @@ -- khóa hoặc được index dùng `VARCHAR(700)` (đủ ngắn để nằm dưới giới hạn -- index key 3072 bytes với utf8mb4, kể cả PK ghép với repo_id). Trường hợp -- key dài hơn 700 ký tự → dùng hash key (md5/sha256) ở phase sau. --- * `COLLATE utf8mb4_bin` giữ so sánh chính xác theo byte (name/path là key --- case-sensitive; collation mặc định *_ci sẽ gộp 'Foo'/'foo'). +-- * `COLLATE utf8mb4_0900_as_cs` giữ so sánh case-sensitive (name/path là key +-- phân biệt hoa/thường) NHƯNG vẫn là charset utf8mb4 (không phải binary) — +-- sqlx decode VARCHAR/TEXT thành String được. Collations *_bin bị MySQL báo +-- về client dưới dạng VARBINARY nên sqlx từ chối decode thành String. -- * id dùng BIGINT signed như sqlite hiện tại (u64 → i64, không đổi hành vi). -- ============================================================================= @@ -19,7 +21,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations ( version VARCHAR(64) NOT NULL PRIMARY KEY, applied_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- ═══════════════════════════════════════════════════════════════════════════ -- Entity store (sg_*) — partition theo repo_id @@ -40,12 +42,12 @@ CREATE TABLE IF NOT EXISTS sg_symbols ( end_line INT NOT NULL DEFAULT 0, signature TEXT, doc TEXT, - annotations JSON NOT NULL, -- app luôn ghi giá trị (JSON không cho DEFAULT) + annotations TEXT NOT NULL, -- lưu JSON string (app luôn ghi giá trị nên không cần DEFAULT) language VARCHAR(64) NOT NULL DEFAULT '', PRIMARY KEY (repo_id, id), KEY idx_sg_symbols_repo_file (repo_id, file), KEY idx_sg_symbols_repo_name (repo_id, name) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- FileInfo — metadata file đã index. -- `lines` được quote vì LINES là reserved word trong MySQL (LOAD DATA ... LINES). @@ -56,7 +58,7 @@ CREATE TABLE IF NOT EXISTS sg_files ( bytes BIGINT NOT NULL DEFAULT 0, `lines` INT NOT NULL DEFAULT 0, PRIMARY KEY (repo_id, path) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Call records của từng function — JSON bytes của `Vec`. CREATE TABLE IF NOT EXISTS sg_call_records ( @@ -64,7 +66,7 @@ CREATE TABLE IF NOT EXISTS sg_call_records ( func BIGINT NOT NULL, -- caller symbol id records LONGBLOB NOT NULL, -- serde_json bytes PRIMARY KEY (repo_id, func) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Inverted index call name → call sites — JSON bytes của `Vec`. CREATE TABLE IF NOT EXISTS sg_call_names ( @@ -72,21 +74,21 @@ CREATE TABLE IF NOT EXISTS sg_call_names ( name VARCHAR(700) NOT NULL, -- tên call (lowercase) sites LONGBLOB NOT NULL, -- serde_json bytes PRIMARY KEY (repo_id, name) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Version index của repo — `SharedGraphIndex::ensure_fresh` probe ở đây. CREATE TABLE IF NOT EXISTS sg_meta ( repo_id BIGINT NOT NULL, version BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (repo_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Registry counter — symbol id tiếp theo (SYMBOL_BASE = 100). CREATE TABLE IF NOT EXISTS sg_next_id ( repo_id BIGINT NOT NULL, next BIGINT NOT NULL DEFAULT 100, -- SYMBOL_BASE PRIMARY KEY (repo_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- ═══════════════════════════════════════════════════════════════════════════ -- Radix trie (rt_*) — partition theo repo_id, giữ nguyên sharding element % 64 @@ -99,7 +101,7 @@ CREATE TABLE IF NOT EXISTS rt_nodes ( prefix LONGBLOB NOT NULL, record BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (repo_id, id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Cạnh cha-con của trie. CREATE TABLE IF NOT EXISTS rt_children ( @@ -108,7 +110,7 @@ CREATE TABLE IF NOT EXISTS rt_children ( child BIGINT NOT NULL, PRIMARY KEY (repo_id, parent, child), KEY idx_rt_children_repo_parent (repo_id, parent) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Gốc mỗi shard: shard ∈ [0, 64). root = 0 nghĩa EMPTY — row tạo LAZY lần đầu -- dùng shard (giống sqlite: get_root trả EMPTY khi thiếu row). @@ -117,7 +119,7 @@ CREATE TABLE IF NOT EXISTS rt_roots ( shard INT NOT NULL, root BIGINT NOT NULL DEFAULT 0, -- EMPTY = 0 PRIMARY KEY (repo_id, shard) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Metadata opaque theo record (call-site info v.v.). CREATE TABLE IF NOT EXISTS rt_meta ( @@ -125,7 +127,7 @@ CREATE TABLE IF NOT EXISTS rt_meta ( record BIGINT NOT NULL, meta LONGBLOB, PRIMARY KEY (repo_id, record) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Độ dài key (số element) theo record — filter depth trong search. CREATE TABLE IF NOT EXISTS rt_keylen ( @@ -133,7 +135,7 @@ CREATE TABLE IF NOT EXISTS rt_keylen ( record BIGINT NOT NULL, len INT NOT NULL, PRIMARY KEY (repo_id, record) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Shortcut index (substring search): node có prefix chứa elem → ứng viên KMP. CREATE TABLE IF NOT EXISTS rt_shortcuts ( @@ -143,7 +145,7 @@ CREATE TABLE IF NOT EXISTS rt_shortcuts ( node_id BIGINT NOT NULL, PRIMARY KEY (repo_id, shard, elem(255), node_id), KEY idx_rt_shortcuts_repo_shard_elem (repo_id, shard, elem(255)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Chain của function (record → bytes u64 LE mỗi element). Nguồn chân lý để -- rebuild engine khi reopen (`GraphIndex::rebuild` → `all_chains()`). @@ -152,7 +154,7 @@ CREATE TABLE IF NOT EXISTS rt_chains ( record BIGINT NOT NULL, -- func id chain LONGBLOB NOT NULL, PRIMARY KEY (repo_id, record) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Edge data stream (legacy — Storage trait còn giữ, GraphIndex chưa dùng). CREATE TABLE IF NOT EXISTS rt_edges ( @@ -160,7 +162,7 @@ CREATE TABLE IF NOT EXISTS rt_edges ( id BIGINT NOT NULL, data LONGBLOB, PRIMARY KEY (repo_id, id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Node metadata stream (Node JSON theo element — legacy, chưa dùng). CREATE TABLE IF NOT EXISTS rt_node_meta ( @@ -168,7 +170,7 @@ CREATE TABLE IF NOT EXISTS rt_node_meta ( elem BIGINT NOT NULL, meta LONGBLOB, PRIMARY KEY (repo_id, elem) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Bloom filter per node (feature `bloom-search`). CREATE TABLE IF NOT EXISTS rt_node_blooms ( @@ -176,14 +178,14 @@ CREATE TABLE IF NOT EXISTS rt_node_blooms ( id BIGINT NOT NULL, bloom LONGBLOB, PRIMARY KEY (repo_id, id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Node-id allocator (per repo — các shard dùng chung một dãy id như sqlite). CREATE TABLE IF NOT EXISTS rt_counter ( repo_id BIGINT NOT NULL, next BIGINT NOT NULL DEFAULT 1, PRIMARY KEY (repo_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- ═══════════════════════════════════════════════════════════════════════════ -- Pattern dùng chung (thực thi ở tầng storage — KHÔNG nằm trong migration, diff --git a/sql/mysql/002-add-repos-registry.sql b/sql/mysql/002-add-repos-registry.sql index c2d4b220b..29a3f4c3e 100644 --- a/sql/mysql/002-add-repos-registry.sql +++ b/sql/mysql/002-add-repos-registry.sql @@ -37,6 +37,6 @@ CREATE TABLE IF NOT EXISTS repos ( shard INT NOT NULL, -- shard server được gán (index vào dsns) root VARCHAR(700), -- root path chuẩn để lookup (nullable: route có thể không có root) created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs; -- Lookup theo root (adopt cùng repo_id cho clone/máy khác). CREATE INDEX idx_repos_root ON repos (root); \ No newline at end of file diff --git a/sql/postgres/001-initial-schema.sql b/sql/postgres/001-initial-schema.sql index 2e2b62ee8..da2053ec7 100644 --- a/sql/postgres/001-initial-schema.sql +++ b/sql/postgres/001-initial-schema.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS sg_symbols ( end_line INTEGER NOT NULL DEFAULT 0, signature TEXT, doc TEXT, - annotations JSONB NOT NULL DEFAULT '[]'::jsonb, + annotations TEXT NOT NULL DEFAULT '[]', -- lưu JSON string (app ghi/đọc bằng serde_json to_string/from_str) language TEXT NOT NULL DEFAULT '', PRIMARY KEY (repo_id, id) ); From 8817f720ec3686a461482aa041d8aba1578e6a61 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 15 Aug 2026 21:17:43 +0700 Subject: [PATCH 8/9] Fix tests --- crates/codegraph-graph/tests/redis.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/codegraph-graph/tests/redis.rs b/crates/codegraph-graph/tests/redis.rs index 673d67362..f2da02e6e 100644 --- a/crates/codegraph-graph/tests/redis.rs +++ b/crates/codegraph-graph/tests/redis.rs @@ -68,6 +68,12 @@ async fn redis_ingest_reopen_roundtrip() { return; } }; + // Use DB 1 to isolate from other test (DB 2) and internal tests (DB 15) + let dsn = if dsn.contains('/') && dsn.rsplit('/').next().unwrap().parse::().is_ok() { + dsn // already has DB number + } else { + format!("{}/1", dsn.trim_end_matches('/')) + }; let calls = vec![CallRecord { caller_id: SYMBOL_BASE, @@ -127,6 +133,12 @@ async fn redis_empty_ingest_wipes_store() { return; } }; + // Use DB 2 to isolate from other test (DB 1) + let dsn = if dsn.contains('/') && dsn.rsplit('/').next().unwrap().parse::().is_ok() { + dsn // already has DB number + } else { + format!("{}/2", dsn.trim_end_matches('/')) + }; let r = result( "a.ts", From c947c12b833899ef2215365c9d74384fadaa6a40 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Sat, 15 Aug 2026 21:24:23 +0700 Subject: [PATCH 9/9] Fix lint --- crates/codegraph/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 63e2eb4bf..b9e87ed4c 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -238,6 +238,7 @@ fn cmd_deinit(root: &Utf8Path) -> Result<()> { /// `codegraph serve --mcp`: chạy MCP server trên stdio. /// `codegraph serve --http`: chạy MCP server trên Streamable HTTP. +#[allow(clippy::too_many_arguments)] async fn cmd_serve( root: &Utf8Path, mcp: bool,