Read non-Transit attachments in encrypted graphs

docs/agent-guide/implemented/bugfix/2026-10-20-encrypted-graph-raw-assets.md

@@ -0,0 +1,39 @@
+# Non-Transit assets in encrypted graphs
+
+The user authorized reading non-Transit attachments in encrypted DB graphs
+through the existing plaintext asset path. This standalone change ports that
+behavior onto main without the separate asset-diagnostic carrier, shape scanner,
+format hints, response-header evidence or typed failure-channel changes.
+
+The public asset codec owns response-byte classification; pure reducer events
+contain no response bytes and cannot reproduce this boundary. A bounded lexical
+check reserves JSON containers, strings, number starts and complete/incomplete
+literal candidates for the existing bounded envelope validator. Leading
+whitespace or BOM cannot disguise such a candidate. Clear non-Transit bytes use
+the unchanged common plaintext maximum-length and SHA-256 checks, then the
+unchanged safe cache publication. JSON-looking raw files remain conservative.
+
+Malformed/truncated candidates, invalid envelope types/IV/ciphertext, parser
+errors, AES authentication failures, crypto exceptions and decrypted-checksum
+failures never become raw-file signals. A matching raw-response checksum does
+not bypass these failures. No catch-all fallback is added. The sole new log is
+the fixed text `journal asset decode path=raw-fallback`, which identifies path
+selection before validation and contains no content, identity, hash or secret.
+
+Public codec regressions cover binary/text/empty raw success, raw limits and
+checksum failures, unchanged valid encryption, malformed/truncated envelopes,
+wrong-key/auth/crypto-exception/decrypted-checksum failures using matching raw
+checksums, whitespace/BOM boundaries and hostile nesting. They are first run
+against main before implementation, then against the standalone final source.
+Existing Runner/cache regressions continue to cover unchanged publication.
+
+Build57 (0d05d1a951f5ccf3ea2c1053401e9629c67be712) was installed and launched on
+the iPhone, but includes additional diagnostic work. That is not device
+acceptance of this standalone PR head, nor proof of successful re-download of
+the originally failed asset. Actual 56051-byte incident content remains unknown.
+No device operation or Apple build is needed for this PR split. No graph flag,
+key ownership, upload, HTTP, retry, cache policy, spec, Dune or dependency changes.
+Temporary validation and review evidence stays outside the repository.
+
+`spec-dev-tool` is unavailable on this Mac; no automated document-schema
+validation is claimed.

logseq_sync/lib/effect_runner/protocol/asset_codec.ml

@@ -60,6 +60,56 @@ let envelope_shape wire =
      if whitespace index = length then Some () else None)
 ;;
 
+(* Reserve JSON/Transit roots, including incomplete envelopes, for the existing
+   bounded decoder. A parse/authentication failure is never a raw-file signal. *)
+let transit_candidate wire =
+  let length = String.length wire in
+  let whitespace = function
+    | ' ' | '\n' | '\r' | '\t' -> true
+    | _ -> false
+  in
+  let rec start index =
+    if index < length && whitespace wire.[index]
+    then start (index + 1)
+    else if
+      index + 3 <= length
+      && wire.[index] = '\239'
+      && wire.[index + 1] = '\187'
+      && wire.[index + 2] = '\191'
+    then start (index + 3)
+    else index
+  in
+  let index = start 0 in
+  let literal text =
+    let count = String.length text in
+    let rec prefix offset =
+      if index + offset = length
+      then true
+      else if offset = count
+      then (
+        match wire.[index + offset] with
+        | ']' | '}' | ',' -> true
+        | c -> whitespace c)
+      else wire.[index + offset] = text.[offset] && prefix (offset + 1)
+    in
+    prefix 0
+  in
+  index < length
+  &&
+  match wire.[index] with
+  | '[' | '{' | '"' | '0' .. '9' -> true
+  | '-' ->
+    index + 1 = length
+    ||
+      (match wire.[index + 1] with
+      | '0' .. '9' -> true
+      | _ -> false)
+  | 't' -> literal "true"
+  | 'f' -> literal "false"
+  | 'n' -> literal "null"
+  | _ -> false
+;;
+
 let decode ~maximum_plaintext_bytes ~crypto ~key ~expected_checksum wire =
   if not (limit maximum_plaintext_bytes)
   then Error "Invalid plaintext size limit"
@@ -75,6 +125,10 @@ let decode ~maximum_plaintext_bytes ~crypto ~key ~expected_checksum wire =
       let* plaintext =
         match key with
         | None -> Ok wire
+        | Some _ when not (transit_candidate wire) ->
+          (try Printf.eprintf "journal asset decode path=raw-fallback\n%!" with
+           | _ -> ());
+          Ok wire
         | Some key ->
           let* () =
             if envelope_shape wire then Ok () else Error "Invalid asset AES-GCM envelope"

logseq_sync/test/test_asset_codec.ml

@@ -101,11 +101,207 @@ let bounded_envelope_shape () =
     (get (decode (" \n" ^ envelope ^ "\t ")))
 ;;
 
+(* HTTP bytes are opaque to the pure reducer. The public codec is the narrowest
+   owner that can reproduce content classification and validation. *)
+let capture_stderr f =
+  let saved = Unix.dup Unix.stderr in
+  let input, output = Unix.pipe () in
+  Unix.dup2 output Unix.stderr;
+  Unix.close output;
+  let outcome =
+    try
+      Ok
+        (Fun.protect
+           ~finally:(fun () ->
+             flush stderr;
+             Unix.dup2 saved Unix.stderr;
+             Unix.close saved)
+           f)
+    with
+    | exn -> Error exn
+  in
+  let channel = Unix.in_channel_of_descr input in
+  let log =
+    Fun.protect
+      ~finally:(fun () -> close_in channel)
+      (fun () -> In_channel.input_all channel)
+  in
+  match outcome with
+  | Ok result -> result, log
+  | Error exn -> raise exn
+;;
+
+let non_transit_raw_assets () =
+  let calls = ref 0 in
+  let guarded =
+    { crypto with
+      decrypt =
+        (fun ~key:_ ~iv:_ ~ciphertext:_ ->
+          incr calls;
+          Error "PRIVATE_UNEXPECTED_DECRYPT")
+    }
+  in
+  List.iter
+    (fun wire ->
+       let result, log =
+         capture_stderr (fun () ->
+           C.decode
+             ~maximum_plaintext_bytes:256
+             ~crypto:guarded
+             ~key:(Some "PRIVATE_KEY")
+             ~expected_checksum:(C.checksum wire)
+             wire)
+       in
+       Alcotest.(check string) "raw bytes preserved" wire (get result);
+       Alcotest.(check string)
+         "fixed path evidence contains no content/key/hash"
+         "journal asset decode path=raw-fallback\n"
+         log)
+    [ bytes
+    ; "\137PNG\r\n\026\nPRIVATE_IMAGE"
+    ; "\255\000PRIVATE_BINARY"
+    ; "not transit PRIVATE_TEXT"
+    ; "hello PRIVATE_TEXT"
+    ; "- PRIVATE_MARKDOWN"
+    ; "\239\187\191PRIVATE_TEXT"
+    ; " \nPRIVATE_TEXT"
+    ; ""
+    ; " \n\t"
+    ];
+  Alcotest.(check int) "raw never decrypts" 0 !calls;
+  Alcotest.(check string) "existing plain graph path" bytes (get (decode ~key:None bytes))
+;;
+
+let raw_assets_keep_validation () =
+  let check ~maximum ~expected wire error =
+    let result, _ =
+      capture_stderr (fun () ->
+        C.decode
+          ~maximum_plaintext_bytes:maximum
+          ~crypto
+          ~key:(Some "key")
+          ~expected_checksum:expected
+          wire)
+    in
+    Alcotest.(check (result string string))
+      "common validation is retained"
+      (Error error)
+      result
+  in
+  check ~maximum:32 ~expected:(String.make 64 'a') bytes "Asset checksum mismatch";
+  check ~maximum:2 ~expected:(C.checksum bytes) bytes "Asset exceeds plaintext size limit";
+  check ~maximum:0 ~expected:(C.checksum "x") "x" "Asset exceeds plaintext size limit";
+  check ~maximum:(-1) ~expected:(C.checksum bytes) bytes "Invalid plaintext size limit";
+  let oversized = String.make 1000 'X' in
+  check
+    ~maximum:0
+    ~expected:(C.checksum oversized)
+    oversized
+    "Asset exceeds wire size limit";
+  let result, _ =
+    capture_stderr (fun () ->
+      decode ~limit:(String.length bytes) ~checksum:(C.checksum bytes) bytes)
+  in
+  Alcotest.(check string) "raw size boundary succeeds" bytes (get result)
+;;
+
+let transit_failures_never_become_raw () =
+  let calls = ref 0 in
+  let guarded =
+    { crypto with
+      decrypt =
+        (fun ~key:_ ~iv:_ ~ciphertext:_ ->
+          incr calls;
+          Error "authentication failed")
+    }
+  in
+  let reject ?(decrypts = false) ?(crypt = guarded) wire =
+    let before = !calls in
+    let result, log =
+      capture_stderr (fun () ->
+        C.decode
+          ~maximum_plaintext_bytes:1048576
+          ~crypto:crypt
+          ~key:(Some "wrong")
+          ~expected_checksum:(C.checksum wire)
+          wire)
+    in
+    rejects result;
+    Alcotest.(check string) "no raw fallback even when raw checksum matches" "" log;
+    if not decrypts
+    then Alcotest.(check int) "malformed candidates never decrypt" before !calls
+  in
+  List.iter
+    reject
+    [ "["
+    ; "[\"~b"
+    ; "[\"~bAA==\","
+    ; "[\"~bAA==\",\"~bAA==\"]"
+    ; "[\"~b!\",\"~b!\"]"
+    ; "[\"a\",\"b\"]"
+    ; "[]"
+    ; "[1,2,3]"
+    ; "{\"PRIVATE\":1}"
+    ; "\"PRIVATE\""
+    ; "null"
+    ; "true"
+    ; "123"
+    ; "tru"
+    ; "nul"
+    ; "-"
+    ; "1e"
+    ; "[\"\\q\",0]"
+    ; envelope ^ "null"
+    ; "\239\187\191" ^ envelope
+    ; " \n\239\187\191" ^ envelope
+    ; "\239\187\191\239\187\191" ^ envelope
+    ; String.sub envelope 0 (String.length envelope - 1)
+    ; T.to_string (Transit_core.Json.Array [ Binary "short"; Binary ciphertext ])
+    ; T.to_string (Transit_core.Json.Array [ Binary iv; Binary "short" ])
+    ; String.make 100000 '[' ^ "0" ^ String.make 100000 ']'
+    ];
+  reject ~decrypts:true envelope;
+  Alcotest.(check int) "valid envelope reaches auth exactly once" 1 !calls;
+  reject
+    ~crypt:{ crypto with decrypt = (fun ~key:_ ~iv:_ ~ciphertext:_ -> Ok "wrong") }
+    envelope;
+  let raised =
+    try
+      reject
+        ~crypt:
+          { crypto with
+            decrypt =
+              (fun ~key:_ ~iv:_ ~ciphertext:_ -> failwith "PRIVATE_AUTH_EXCEPTION")
+          }
+        envelope;
+      false
+    with
+    | Failure message -> message = "PRIVATE_AUTH_EXCEPTION"
+  in
+  Alcotest.(check bool) "crypto exception is not a plaintext signal" true raised;
+  Alcotest.(check string)
+    "valid encrypted success remains unchanged"
+    bytes
+    (get (decode envelope))
+;;
+
 let () =
   Alcotest.run
     "asset codec"
     [ ( "binary protocol"
-      , [ Alcotest.test_case "raw binary envelope" `Quick raw_bytes
+      , [ Alcotest.test_case
+            "non-Transit assets in encrypted graphs"
+            `Quick
+            non_transit_raw_assets
+        ; Alcotest.test_case
+            "raw assets preserve validation"
+            `Quick
+            raw_assets_keep_validation
+        ; Alcotest.test_case
+            "Transit and crypto failures never fall back"
+            `Quick
+            transit_failures_never_become_raw
+        ; Alcotest.test_case "raw binary envelope" `Quick raw_bytes
         ; Alcotest.test_case "plaintext" `Quick plaintext
         ; Alcotest.test_case "invalid and oversized" `Quick validation
         ; Alcotest.test_case "bounded envelope shape" `Quick bounded_envelope_shape