diff --git a/docs/internals/packs.rst b/docs/internals/packs.rst index 5f97d22f34..c601d4817d 100644 --- a/docs/internals/packs.rst +++ b/docs/internals/packs.rst @@ -34,7 +34,7 @@ Each blob is a self-contained unit:: Offset (relative to blob start) Size Type Field -------------------------------- ---------------- ------- ----- 0 len(OBJ_MAGIC) bytes OBJ_MAGIC = ASCII b"BORG_OBJ" - 8 1 uint8 Format version: 0x02 (0x01 still readable) + 8 1 uint8 Format version: 0x02 9 32 bytes chunk_id 41 4 uint32le meta_size 45 4 uint32le data_size @@ -56,7 +56,8 @@ The fixed part of each blob header is 49 bytes (``REPOOBJ_HEADER_SIZE``): ``len(OBJ_MAGIC)`` + 1 version + 32 chunk_id + 4 meta_size + 4 data_size. ``REPOOBJ_HEADER_SIZE = len(OBJ_MAGIC) + 1 + 32 + 4 + 4 = 49`` -Format version ``0x02`` (``OBJ_VERSION_HEADER_AAD``) binds the header's first 41 bytes (``OBJ_MAGIC`` +The format version is ``0x02`` (``OBJ_VERSION_HEADER_AAD``), the only version ``RepoObj.format()`` +writes and ``parse()``/``parse_meta()`` accept. It binds the header's first 41 bytes (``OBJ_MAGIC`` + version + ``chunk_id`` -- ``REPOOBJ_HEADER_AAD_SIZE``) into the authentication of ``encrypted_meta`` and ``encrypted_data`` as additional authenticated data (AAD: data that is authenticated together with the ciphertext, but not itself encrypted). This applies to all borg 2 @@ -75,10 +76,6 @@ its slot. This stops an attacker controlling repo storage from swapping the two ``meta_size``/``data_size`` to match): decrypting a ciphertext under the wrong slot's AAD fails authentication. -Format version ``0x01`` (``OBJ_VERSION_NO_HEADER_AAD``) authenticates ``encrypted_meta`` and -``encrypted_data`` with ``aad=chunk_id`` only, without the header bound in. ``RepoObj.format()`` -writes version ``0x02``; ``parse()``/``parse_meta()`` accept both versions. - ``iter_headers()`` (used for pack recovery/compaction, see below) reads the header without decrypting, so it does not check header AAD authentication. The repair walk described below is the exception: given a validator it reads and decrypts each metadata slot, and thus does check it. @@ -90,8 +87,8 @@ exception: given a validator it reads and decrypts each metadata slot, and thus The fixed 49-byte blob header. ``meta_size`` and ``data_size`` drive traversal; integrity comes from the content-addressed pack name and the - per-blob tag, which at version ``0x02`` authenticates magic/version/chunk_id - as additional authenticated data. + per-blob tag, which authenticates magic/version/chunk_id as additional + authenticated data. A reader locates the next blob by advancing:: diff --git a/src/borg/repoobj.py b/src/borg/repoobj.py index 385001d6c8..82d113f25d 100644 --- a/src/borg/repoobj.py +++ b/src/borg/repoobj.py @@ -60,25 +60,22 @@ def get_assert_id_places(): OBJ_MAGIC = b"BORG_OBJ" -# meta_encrypted/data_encrypted are AEAD-authenticated with aad=chunk_id. -OBJ_VERSION_NO_HEADER_AAD = 0x01 # meta_encrypted/data_encrypted are AEAD-authenticated with aad=header_aad+slot_tag+chunk_id. header_aad # is the header prefix (magic, version, chunk_id; REPOOBJ_HEADER_AAD_SIZE bytes). slot_tag is b"M" for # meta_encrypted, b"D" for data_encrypted, binding each ciphertext to its slot. format() writes this version. OBJ_VERSION_HEADER_AAD = 0x02 OBJ_VERSION = OBJ_VERSION_HEADER_AAD # Versions accepted by parse() and parse_meta(). -SUPPORTED_OBJ_VERSIONS = (OBJ_VERSION_NO_HEADER_AAD, OBJ_VERSION_HEADER_AAD) +SUPPORTED_OBJ_VERSIONS = (OBJ_VERSION_HEADER_AAD,) # Fixed header size per blob: OBJ_MAGIC(8) + version(1) + chunk_id(32) + meta_size(4) + data_size(4) REPOOBJ_HEADER_SIZE = 49 # Size of the header prefix used as AEAD AAD (additional authenticated data: authenticated together -# with the ciphertext, but not itself encrypted) for OBJ_VERSION_HEADER_AAD objects: magic(8) + -# version(1) + chunk_id(32). meta_size and data_size are excluded, since they are only known after -# encryption. A change to either changes the ciphertext slice length, so parse(), which reads both -# slots, fails authentication; parse_meta() reads the metadata slot alone and thus does not see a -# changed data_size. +# with the ciphertext, but not itself encrypted): magic(8) + version(1) + chunk_id(32). meta_size and +# data_size are excluded, since they are only known after encryption. A change to either changes the +# ciphertext slice length, so parse(), which reads both slots, fails authentication; parse_meta() +# reads the metadata slot alone and thus does not see a changed data_size. REPOOBJ_HEADER_AAD_SIZE = len(OBJ_MAGIC) + 1 + 32 META_AAD_TAG = b"M" @@ -184,9 +181,9 @@ def parse_meta(self, id: bytes, cdata: bytes | memoryview, ro_type: str) -> dict raise IntegrityError( f"object too small: expected at least {hdr_size + hdr.meta_size} bytes, got {len(obj)}" ) - # header_aad, meta_aad: see OBJ_VERSION_HEADER_AAD above. b"" for OBJ_VERSION_NO_HEADER_AAD. - header_aad = bytes(obj[:REPOOBJ_HEADER_AAD_SIZE]) if hdr.version == OBJ_VERSION_HEADER_AAD else b"" - meta_aad = header_aad + META_AAD_TAG if hdr.version == OBJ_VERSION_HEADER_AAD else header_aad + # header_aad, meta_aad: see OBJ_VERSION_HEADER_AAD above. + header_aad = bytes(obj[:REPOOBJ_HEADER_AAD_SIZE]) + meta_aad = header_aad + META_AAD_TAG meta_encrypted = obj[hdr_size : hdr_size + hdr.meta_size] meta_packed = self.key.decrypt(id, meta_encrypted, aad=meta_aad) meta = msgpack.unpackb(meta_packed) @@ -233,10 +230,10 @@ def parse( overall_expected_size = hdr_size + hdr.meta_size + hdr.data_size if overall_expected_size != len(obj): raise IntegrityError(f"object size inconsistent: expected {overall_expected_size} bytes, got {len(obj)}") - # header_aad, meta_aad: see parse_meta(). - header_aad = bytes(obj[:REPOOBJ_HEADER_AAD_SIZE]) if hdr.version == OBJ_VERSION_HEADER_AAD else b"" - meta_aad = header_aad + META_AAD_TAG if hdr.version == OBJ_VERSION_HEADER_AAD else header_aad - data_aad = header_aad + DATA_AAD_TAG if hdr.version == OBJ_VERSION_HEADER_AAD else header_aad + # header_aad, meta_aad, data_aad: see OBJ_VERSION_HEADER_AAD above. + header_aad = bytes(obj[:REPOOBJ_HEADER_AAD_SIZE]) + meta_aad = header_aad + META_AAD_TAG + data_aad = header_aad + DATA_AAD_TAG meta_encrypted = obj[hdr_size : hdr_size + hdr.meta_size] meta_packed = self.key.decrypt(id, meta_encrypted, aad=meta_aad) meta_compressed = msgpack.unpackb(meta_packed) # means: before adding more metadata in decompress block @@ -284,13 +281,11 @@ def object_validator(repo_objs): """Return validate(chunk_id, obj): True if obj is the repo object with id chunk_id. obj is an object's header plus its metadata slot. Parsing that slot verifies its tag, which is - computed over the slot itself and over the chunk id, so a wrong meta_size or chunk id fails it. - At object version OBJ_VERSION_HEADER_AAD the magic and the version are covered as well (as AAD, - additional authenticated data: bytes the tag covers without being part of the ciphertext); at - OBJ_VERSION_NO_HEADER_AAD they are not: a wrong magic fails the explicit magic check, and a - wrong version fails because the version selects the AAD the slot is parsed with. data_size, the - one header field outside the tag at either version, must match csize - the data slot's payload - size, recorded in the tagged metadata - plus the key's fixed envelope overhead. + computed over the slot itself, over the chunk id and over the header prefix (magic, version, + chunk id) as AAD (additional authenticated data: bytes the tag covers without being part of the + ciphertext), so a wrong magic, version, chunk id or meta_size fails it. data_size, the one header + field outside the tag, must match csize - the data slot's payload size, recorded in the tagged + metadata - plus the key's fixed envelope overhead. In the "none-*" modes the tag is an unkeyed checksum, and in the "authenticated-*" modes it is deterministic and binds an object to its chunk id alone. Both therefore accept an object that a diff --git a/src/borg/testsuite/repoobj_test.py b/src/borg/testsuite/repoobj_test.py index 89232595b4..582f3de1e9 100644 --- a/src/borg/testsuite/repoobj_test.py +++ b/src/borg/testsuite/repoobj_test.py @@ -11,8 +11,6 @@ BORG_ASSERT_ID_DEFAULT, OBJ_MAGIC, OBJ_VERSION, - OBJ_VERSION_NO_HEADER_AAD, - REPOOBJ_HEADER_SIZE, RepoObj, object_validator, ) @@ -484,29 +482,6 @@ def test_assert_id_configurable_for_authenticated_key(authenticated_key, monkeyp repo_objs.parse(id, cdata, ro_type=ROBJ_FILE_STREAM, assert_id_place=place) -def test_version1_object_without_header_aad_still_readable(aead_key): - # Builds an OBJ_VERSION_NO_HEADER_AAD object by hand (format() only writes OBJ_VERSION_HEADER_AAD) - # and checks that parse()/parse_meta() still decrypt it. - repo_objs = RepoObj(aead_key) - data = b"foobar" * 10 - id = repo_objs.id_hash(data) - meta = {"type": ROBJ_FILE_STREAM} - meta, data_compressed = repo_objs.compressor.compress(meta, data) - - # OBJ_VERSION_NO_HEADER_AAD encoding: aad=chunk_id only, no header bound in. - data_encrypted = aead_key.encrypt(id, data_compressed, aad=b"") - meta_packed = msgpack.packb(meta) - meta_encrypted = aead_key.encrypt(id, meta_packed, aad=b"") - hdr = RepoObj.ObjHeader(OBJ_MAGIC, OBJ_VERSION_NO_HEADER_AAD, id, len(meta_encrypted), len(data_encrypted)) - cdata = RepoObj.obj_header.pack(*hdr) + meta_encrypted + data_encrypted - assert len(RepoObj.obj_header.pack(*hdr)) == REPOOBJ_HEADER_SIZE - - got_meta = repo_objs.parse_meta(id, cdata, ro_type=ROBJ_FILE_STREAM) - assert got_meta["type"] == ROBJ_FILE_STREAM - got_meta, got_data = repo_objs.parse(id, cdata, ro_type=ROBJ_FILE_STREAM) - assert got_data == data - - def validator_input(repo_objs, data): # (chunk_id, head) of a real repo object storing data: head is its object header plus its # metadata slot, which is what validate takes.