From c9339d45f0efc7c96f39627754a8071afaa98931 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Tue, 20 Oct 2026 19:46:07 +0800 Subject: [PATCH] fix: read non-Transit assets in encrypted graphs Port the compatibility behavior directly onto main without the unselected asset-diagnostic infrastructure. Clear non-Transit bytes use existing plaintext limits/checksum and unchanged safe publication; Transit candidates and crypto failures never fall back. Keep path evidence fixed and payload-free. Add public codec regressions for success, malformed/truncated candidates, authentication, limits, checksum and exception boundaries. --- .../2026-10-20-encrypted-graph-raw-assets.md | 39 ++++ .../lib/effect_runner/protocol/asset_codec.ml | 54 +++++ logseq_sync/test/test_asset_codec.ml | 198 +++++++++++++++++- 3 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 docs/agent-guide/implemented/bugfix/2026-10-20-encrypted-graph-raw-assets.md diff --git a/docs/agent-guide/implemented/bugfix/2026-10-20-encrypted-graph-raw-assets.md b/docs/agent-guide/implemented/bugfix/2026-10-20-encrypted-graph-raw-assets.md new file mode 100644 index 00000000..bf0bf9d9 --- /dev/null +++ b/docs/agent-guide/implemented/bugfix/2026-10-20-encrypted-graph-raw-assets.md @@ -0,0 +1,39 @@ +# Non-Transit assets in encrypted graphs + +The user authorized reading non-Transit attachments in encrypted DB graphs +through the existing plaintext asset path. This standalone change ports that +behavior onto main without the separate asset-diagnostic carrier, shape scanner, +format hints, response-header evidence or typed failure-channel changes. + +The public asset codec owns response-byte classification; pure reducer events +contain no response bytes and cannot reproduce this boundary. A bounded lexical +check reserves JSON containers, strings, number starts and complete/incomplete +literal candidates for the existing bounded envelope validator. Leading +whitespace or BOM cannot disguise such a candidate. Clear non-Transit bytes use +the unchanged common plaintext maximum-length and SHA-256 checks, then the +unchanged safe cache publication. JSON-looking raw files remain conservative. + +Malformed/truncated candidates, invalid envelope types/IV/ciphertext, parser +errors, AES authentication failures, crypto exceptions and decrypted-checksum +failures never become raw-file signals. A matching raw-response checksum does +not bypass these failures. No catch-all fallback is added. The sole new log is +the fixed text `journal asset decode path=raw-fallback`, which identifies path +selection before validation and contains no content, identity, hash or secret. + +Public codec regressions cover binary/text/empty raw success, raw limits and +checksum failures, unchanged valid encryption, malformed/truncated envelopes, +wrong-key/auth/crypto-exception/decrypted-checksum failures using matching raw +checksums, whitespace/BOM boundaries and hostile nesting. They are first run +against main before implementation, then against the standalone final source. +Existing Runner/cache regressions continue to cover unchanged publication. + +Build57 (0d05d1a951f5ccf3ea2c1053401e9629c67be712) was installed and launched on +the iPhone, but includes additional diagnostic work. That is not device +acceptance of this standalone PR head, nor proof of successful re-download of +the originally failed asset. Actual 56051-byte incident content remains unknown. +No device operation or Apple build is needed for this PR split. No graph flag, +key ownership, upload, HTTP, retry, cache policy, spec, Dune or dependency changes. +Temporary validation and review evidence stays outside the repository. + +`spec-dev-tool` is unavailable on this Mac; no automated document-schema +validation is claimed. diff --git a/logseq_sync/lib/effect_runner/protocol/asset_codec.ml b/logseq_sync/lib/effect_runner/protocol/asset_codec.ml index 16881ab6..4dc52ffe 100644 --- a/logseq_sync/lib/effect_runner/protocol/asset_codec.ml +++ b/logseq_sync/lib/effect_runner/protocol/asset_codec.ml @@ -60,6 +60,56 @@ let envelope_shape wire = if whitespace index = length then Some () else None) ;; +(* Reserve JSON/Transit roots, including incomplete envelopes, for the existing + bounded decoder. A parse/authentication failure is never a raw-file signal. *) +let transit_candidate wire = + let length = String.length wire in + let whitespace = function + | ' ' | '\n' | '\r' | '\t' -> true + | _ -> false + in + let rec start index = + if index < length && whitespace wire.[index] + then start (index + 1) + else if + index + 3 <= length + && wire.[index] = '\239' + && wire.[index + 1] = '\187' + && wire.[index + 2] = '\191' + then start (index + 3) + else index + in + let index = start 0 in + let literal text = + let count = String.length text in + let rec prefix offset = + if index + offset = length + then true + else if offset = count + then ( + match wire.[index + offset] with + | ']' | '}' | ',' -> true + | c -> whitespace c) + else wire.[index + offset] = text.[offset] && prefix (offset + 1) + in + prefix 0 + in + index < length + && + match wire.[index] with + | '[' | '{' | '"' | '0' .. '9' -> true + | '-' -> + index + 1 = length + || + (match wire.[index + 1] with + | '0' .. '9' -> true + | _ -> false) + | 't' -> literal "true" + | 'f' -> literal "false" + | 'n' -> literal "null" + | _ -> false +;; + let decode ~maximum_plaintext_bytes ~crypto ~key ~expected_checksum wire = if not (limit maximum_plaintext_bytes) then Error "Invalid plaintext size limit" @@ -75,6 +125,10 @@ let decode ~maximum_plaintext_bytes ~crypto ~key ~expected_checksum wire = let* plaintext = match key with | None -> Ok wire + | Some _ when not (transit_candidate wire) -> + (try Printf.eprintf "journal asset decode path=raw-fallback\n%!" with + | _ -> ()); + Ok wire | Some key -> let* () = if envelope_shape wire then Ok () else Error "Invalid asset AES-GCM envelope" diff --git a/logseq_sync/test/test_asset_codec.ml b/logseq_sync/test/test_asset_codec.ml index a37ec05f..06f88a3a 100644 --- a/logseq_sync/test/test_asset_codec.ml +++ b/logseq_sync/test/test_asset_codec.ml @@ -101,11 +101,207 @@ let bounded_envelope_shape () = (get (decode (" \n" ^ envelope ^ "\t "))) ;; +(* HTTP bytes are opaque to the pure reducer. The public codec is the narrowest + owner that can reproduce content classification and validation. *) +let capture_stderr f = + let saved = Unix.dup Unix.stderr in + let input, output = Unix.pipe () in + Unix.dup2 output Unix.stderr; + Unix.close output; + let outcome = + try + Ok + (Fun.protect + ~finally:(fun () -> + flush stderr; + Unix.dup2 saved Unix.stderr; + Unix.close saved) + f) + with + | exn -> Error exn + in + let channel = Unix.in_channel_of_descr input in + let log = + Fun.protect + ~finally:(fun () -> close_in channel) + (fun () -> In_channel.input_all channel) + in + match outcome with + | Ok result -> result, log + | Error exn -> raise exn +;; + +let non_transit_raw_assets () = + let calls = ref 0 in + let guarded = + { crypto with + decrypt = + (fun ~key:_ ~iv:_ ~ciphertext:_ -> + incr calls; + Error "PRIVATE_UNEXPECTED_DECRYPT") + } + in + List.iter + (fun wire -> + let result, log = + capture_stderr (fun () -> + C.decode + ~maximum_plaintext_bytes:256 + ~crypto:guarded + ~key:(Some "PRIVATE_KEY") + ~expected_checksum:(C.checksum wire) + wire) + in + Alcotest.(check string) "raw bytes preserved" wire (get result); + Alcotest.(check string) + "fixed path evidence contains no content/key/hash" + "journal asset decode path=raw-fallback\n" + log) + [ bytes + ; "\137PNG\r\n\026\nPRIVATE_IMAGE" + ; "\255\000PRIVATE_BINARY" + ; "not transit PRIVATE_TEXT" + ; "hello PRIVATE_TEXT" + ; "- PRIVATE_MARKDOWN" + ; "\239\187\191PRIVATE_TEXT" + ; " \nPRIVATE_TEXT" + ; "" + ; " \n\t" + ]; + Alcotest.(check int) "raw never decrypts" 0 !calls; + Alcotest.(check string) "existing plain graph path" bytes (get (decode ~key:None bytes)) +;; + +let raw_assets_keep_validation () = + let check ~maximum ~expected wire error = + let result, _ = + capture_stderr (fun () -> + C.decode + ~maximum_plaintext_bytes:maximum + ~crypto + ~key:(Some "key") + ~expected_checksum:expected + wire) + in + Alcotest.(check (result string string)) + "common validation is retained" + (Error error) + result + in + check ~maximum:32 ~expected:(String.make 64 'a') bytes "Asset checksum mismatch"; + check ~maximum:2 ~expected:(C.checksum bytes) bytes "Asset exceeds plaintext size limit"; + check ~maximum:0 ~expected:(C.checksum "x") "x" "Asset exceeds plaintext size limit"; + check ~maximum:(-1) ~expected:(C.checksum bytes) bytes "Invalid plaintext size limit"; + let oversized = String.make 1000 'X' in + check + ~maximum:0 + ~expected:(C.checksum oversized) + oversized + "Asset exceeds wire size limit"; + let result, _ = + capture_stderr (fun () -> + decode ~limit:(String.length bytes) ~checksum:(C.checksum bytes) bytes) + in + Alcotest.(check string) "raw size boundary succeeds" bytes (get result) +;; + +let transit_failures_never_become_raw () = + let calls = ref 0 in + let guarded = + { crypto with + decrypt = + (fun ~key:_ ~iv:_ ~ciphertext:_ -> + incr calls; + Error "authentication failed") + } + in + let reject ?(decrypts = false) ?(crypt = guarded) wire = + let before = !calls in + let result, log = + capture_stderr (fun () -> + C.decode + ~maximum_plaintext_bytes:1048576 + ~crypto:crypt + ~key:(Some "wrong") + ~expected_checksum:(C.checksum wire) + wire) + in + rejects result; + Alcotest.(check string) "no raw fallback even when raw checksum matches" "" log; + if not decrypts + then Alcotest.(check int) "malformed candidates never decrypt" before !calls + in + List.iter + reject + [ "[" + ; "[\"~b" + ; "[\"~bAA==\"," + ; "[\"~bAA==\",\"~bAA==\"]" + ; "[\"~b!\",\"~b!\"]" + ; "[\"a\",\"b\"]" + ; "[]" + ; "[1,2,3]" + ; "{\"PRIVATE\":1}" + ; "\"PRIVATE\"" + ; "null" + ; "true" + ; "123" + ; "tru" + ; "nul" + ; "-" + ; "1e" + ; "[\"\\q\",0]" + ; envelope ^ "null" + ; "\239\187\191" ^ envelope + ; " \n\239\187\191" ^ envelope + ; "\239\187\191\239\187\191" ^ envelope + ; String.sub envelope 0 (String.length envelope - 1) + ; T.to_string (Transit_core.Json.Array [ Binary "short"; Binary ciphertext ]) + ; T.to_string (Transit_core.Json.Array [ Binary iv; Binary "short" ]) + ; String.make 100000 '[' ^ "0" ^ String.make 100000 ']' + ]; + reject ~decrypts:true envelope; + Alcotest.(check int) "valid envelope reaches auth exactly once" 1 !calls; + reject + ~crypt:{ crypto with decrypt = (fun ~key:_ ~iv:_ ~ciphertext:_ -> Ok "wrong") } + envelope; + let raised = + try + reject + ~crypt: + { crypto with + decrypt = + (fun ~key:_ ~iv:_ ~ciphertext:_ -> failwith "PRIVATE_AUTH_EXCEPTION") + } + envelope; + false + with + | Failure message -> message = "PRIVATE_AUTH_EXCEPTION" + in + Alcotest.(check bool) "crypto exception is not a plaintext signal" true raised; + Alcotest.(check string) + "valid encrypted success remains unchanged" + bytes + (get (decode envelope)) +;; + let () = Alcotest.run "asset codec" [ ( "binary protocol" - , [ Alcotest.test_case "raw binary envelope" `Quick raw_bytes + , [ Alcotest.test_case + "non-Transit assets in encrypted graphs" + `Quick + non_transit_raw_assets + ; Alcotest.test_case + "raw assets preserve validation" + `Quick + raw_assets_keep_validation + ; Alcotest.test_case + "Transit and crypto failures never fall back" + `Quick + transit_failures_never_become_raw + ; Alcotest.test_case "raw binary envelope" `Quick raw_bytes ; Alcotest.test_case "plaintext" `Quick plaintext ; Alcotest.test_case "invalid and oversized" `Quick validation ; Alcotest.test_case "bounded envelope shape" `Quick bounded_envelope_shape