From b6728ede7b4771fadab1326815bb4ad2a94ddbaf Mon Sep 17 00:00:00 2001
From: Ethan Woodbury <woodburyethan60@gmail.com>
Date: Wed, 28 Oct 2026 01:30:18 -0700
Subject: [PATCH 1/2] fix(spec): match manifest partition fields by field ID

Closes #3565.
---
 crates/iceberg/src/avro/schema.rs             |   2 +-
 crates/iceberg/src/spec/manifest/mod.rs       | 140 +++++++++++++++++-
 crates/iceberg/src/spec/values/serde.rs       |   5 +-
 crates/iceberg/testdata/manifests/README.md   |  43 ++++++
 .../manifests/java-v1-nested-bucket.avro      | Bin 0 -> 6017 bytes
 .../manifests/java-v2-nested-bucket.avro      | Bin 0 -> 7261 bytes
 6 files changed, 183 insertions(+), 7 deletions(-)
 create mode 100644 crates/iceberg/testdata/manifests/java-v1-nested-bucket.avro
 create mode 100644 crates/iceberg/testdata/manifests/java-v2-nested-bucket.avro

diff --git a/crates/iceberg/src/avro/schema.rs b/crates/iceberg/src/avro/schema.rs
index 9af68022a2..9c87ad94e8 100644
--- a/crates/iceberg/src/avro/schema.rs
+++ b/crates/iceberg/src/avro/schema.rs
@@ -34,7 +34,7 @@ use crate::spec::{
 use crate::{Error, ErrorKind, Result, ensure_data_valid};
 
 const ELEMENT_ID: &str = "element-id";
-const FIELD_ID_PROP: &str = "field-id";
+pub(crate) const FIELD_ID_PROP: &str = "field-id";
 const KEY_ID: &str = "key-id";
 const VALUE_ID: &str = "value-id";
 const MAP_LOGICAL_TYPE: &str = "map";
diff --git a/crates/iceberg/src/spec/manifest/mod.rs b/crates/iceberg/src/spec/manifest/mod.rs
index f4269638a2..d4fb2477f3 100644
--- a/crates/iceberg/src/spec/manifest/mod.rs
+++ b/crates/iceberg/src/spec/manifest/mod.rs
@@ -26,18 +26,19 @@ pub use metadata::*;
 mod reader;
 pub use reader::*;
 mod writer;
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 
-use apache_avro::Reader as AvroReader;
 use apache_avro::error::Details;
+use apache_avro::{Reader as AvroReader, Schema as AvroSchema};
 pub use writer::*;
 
 use super::{
     Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, Schema, Struct, Type,
     UNASSIGNED_SEQUENCE_NUMBER,
 };
-use crate::avro::{Resolved, define_named_types_once};
+use crate::avro::{FIELD_ID_PROP, Resolved, define_named_types_once};
 use crate::error::{Error, Result, invalid_data};
 
 /// Whether a manifest with repeated Avro named type definitions was logged at
@@ -109,7 +110,8 @@ impl Manifest {
         // Wrap the partition type once and share it across all entries: the
         // per-entry conversion needs a `&Type`, and building it here keeps the
         // lazily-populated field-name lookup from being rebuilt for every entry.
-        let partition_struct_type = Type::Struct(partition_type.clone());
+        let partition_struct_type =
+            Type::Struct(with_writer_names(reader.writer_schema(), &partition_type));
 
         let entries = match metadata.format_version {
             FormatVersion::V1 => reader
@@ -169,6 +171,61 @@ impl Manifest {
     }
 }
 
+/// `partition_type` with each field renamed to the writer's Avro name for it,
+/// matched by `field-id`. If the writer's partition fields have no field IDs,
+/// they are matched by name. Java writes a name that isn't a valid Avro name,
+/// such as `a.b`, as `a_x2Eb`.
+fn with_writer_names(
+    writer_schema: &AvroSchema,
+    partition_type: &super::StructType,
+) -> super::StructType {
+    fn field<'a>(schema: &'a AvroSchema, name: &str) -> Option<&'a AvroSchema> {
+        match schema {
+            AvroSchema::Record(record) => {
+                let i = *record.lookup.get(name)?;
+                record.fields.get(i).map(|f| &f.schema)
+            }
+            _ => None,
+        }
+    }
+    let partition_fields = match field(writer_schema, "data_file")
+        .and_then(|data_file| field(data_file, "partition"))
+    {
+        Some(AvroSchema::Record(partition)) => partition.fields.as_slice(),
+        _ => &[],
+    };
+    let names: HashMap<i64, &str> = partition_fields
+        .iter()
+        .filter_map(|f| {
+            Some((
+                f.custom_attributes.get(FIELD_ID_PROP)?.as_i64()?,
+                f.name.as_str(),
+            ))
+        })
+        .collect();
+    if names.is_empty() {
+        return partition_type.clone();
+    }
+    super::StructType::new(
+        partition_type
+            .fields()
+            .iter()
+            .map(|f| {
+                let name = match names.get(&i64::from(f.id)) {
+                    Some(&name) if name == f.name => return f.clone(),
+                    Some(&name) => name.to_string(),
+                    // apache-avro rejects `#` in names, so this matches nothing and reads as null.
+                    None => format!("{}#{}", f.name, f.id),
+                };
+                Arc::new(super::NestedField {
+                    name,
+                    ..f.as_ref().clone()
+                })
+            })
+            .collect(),
+    )
+}
+
 /// Serialize a DataFile to a JSON string.
 pub fn serialize_data_file_to_json(
     data_file: DataFile,
@@ -1876,6 +1933,83 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_parse_manifest_written_by_java_with_sanitized_partition_name() {
+        // Java writes the partition field `user.id_bucket` as `user_x2Eid_bucket`.
+        for (file, format_version) in [
+            ("java-v1-nested-bucket.avro", FormatVersion::V1),
+            ("java-v2-nested-bucket.avro", FormatVersion::V2),
+        ] {
+            let bs = fs::read(format!(
+                "{}/testdata/manifests/{file}",
+                env!("CARGO_MANIFEST_DIR")
+            ))
+            .unwrap();
+
+            let manifest = Manifest::parse_avro(&bs).unwrap();
+
+            assert_eq!(manifest.metadata().format_version, format_version);
+            assert_eq!(
+                manifest.metadata().partition_spec.fields()[0].name,
+                "user.id_bucket"
+            );
+            assert_eq!(
+                *manifest.entries()[0].data_file().partition(),
+                Struct::from_iter([Some(Literal::int(3))])
+            );
+        }
+    }
+
+    #[test]
+    fn test_parse_manifest_matches_partition_fields_by_field_id() {
+        let metadata = writer_schema_test_metadata();
+        let cases = [
+            // In spec order, but `name` and `score` have each other's field IDs.
+            (
+                serde_json::json!([
+                    {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
+                    {"name": "name", "type": ["null", "double"], "default": null, "field-id": 1002},
+                    {"name": "score", "type": ["null", "string"], "default": null, "field-id": 1001},
+                ]),
+                serde_json::json!({"id": 5, "name": 2.5, "score": "a"}),
+                Some(Literal::double(2.5)),
+            ),
+            // With field IDs, a field whose name matches but whose ID doesn't reads as null.
+            (
+                serde_json::json!([
+                    {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
+                    {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
+                    {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1005},
+                ]),
+                serde_json::json!({"id": 5, "name": "a", "score": 2.5}),
+                None,
+            ),
+            // Without field IDs, fields match by name.
+            (
+                serde_json::json!([
+                    {"name": "score", "type": ["null", "double"], "default": null},
+                    {"name": "id", "type": ["null", "long"], "default": null},
+                    {"name": "name", "type": ["null", "string"], "default": null},
+                ]),
+                serde_json::json!({"score": 2.5, "id": 5, "name": "a"}),
+                Some(Literal::double(2.5)),
+            ),
+        ];
+        for (partition_fields, partition, score) in cases {
+            let bs =
+                write_with_writer_schema(&metadata, &v2_writer_schema(partition_fields), vec![
+                    v2_entry(partition),
+                ]);
+
+            let partition =
+                Struct::from_iter([Some(Literal::long(5)), Some(Literal::string("a")), score]);
+            assert_eq!(
+                Manifest::parse_avro(&bs).unwrap(),
+                Manifest::new(metadata.clone(), vec![expected_entry(partition)])
+            );
+        }
+    }
+
     #[test]
     fn test_parse_manifest_promotes_writer_types() {
         // Avro promotes int to long and float to double.
diff --git a/crates/iceberg/src/spec/values/serde.rs b/crates/iceberg/src/spec/values/serde.rs
index 028c2d796e..b8aad07b26 100644
--- a/crates/iceberg/src/spec/values/serde.rs
+++ b/crates/iceberg/src/spec/values/serde.rs
@@ -53,9 +53,8 @@ pub(crate) mod _serde {
         /// record lacks. Record fields that `struct_type` lacks are dropped. Values
         /// other than records are returned unchanged.
         ///
-        /// Fields aren't matched by field ID. A writer that stores a field under
-        /// another name, such as Java replacing characters that Avro names don't
-        /// allow, reads as a missing field.
+        /// Fields aren't matched by field ID; callers that need that rename
+        /// `struct_type`'s fields to the writer's names first.
         pub fn project_by_name(self, struct_type: &StructType) -> Result<Self, Error> {
             let (mut required, optional) = match self.0 {
                 RawLiteralEnum::Record(Record { required, optional }) => (required, optional),
diff --git a/crates/iceberg/testdata/manifests/README.md b/crates/iceberg/testdata/manifests/README.md
index 6b68b71ed0..cabaa5512a 100644
--- a/crates/iceberg/testdata/manifests/README.md
+++ b/crates/iceberg/testdata/manifests/README.md
@@ -97,3 +97,46 @@ def main(path: str) -> None:
 if __name__ == "__main__":
     main(sys.argv[1])
 ```
+
+`java-v1-nested-bucket.avro` and `java-v2-nested-bucket.avro` are V1 and V2 data
+manifests written by iceberg-java 1.12.0. Their table schema has optional columns
+`event` (string, id 1) and `user` (struct, id 2) with an optional `id` (long,
+id 3), and their partition spec is `bucket(16, user.id)`. Java names the
+partition field `user.id_bucket`, which isn't a valid Avro name, so the Avro
+schema stores it as `user_x2Eid_bucket` with `field-id` 1000 and the original in
+`iceberg-field-name`. Each holds one entry for `s3://bucket/table/data/a.parquet`
+with partition value 3.
+
+To regenerate them, run this program against iceberg-java 1.12.0
+(`iceberg-core` and its dependencies), passing the output directory:
+
+```java
+import java.io.File;
+import org.apache.iceberg.*;
+import org.apache.iceberg.types.Types;
+
+public class WriteManifests {
+  public static void main(String[] args) throws Exception {
+    Schema schema = new Schema(
+        Types.NestedField.optional(1, "event", Types.StringType.get()),
+        Types.NestedField.optional(2, "user", Types.StructType.of(
+            Types.NestedField.optional(3, "id", Types.LongType.get()))));
+    PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("user.id", 16).build();
+    for (int version : new int[] {1, 2}) {
+      File out = new File(args[0], "java-v" + version + "-nested-bucket.avro");
+      PartitionData partition = new PartitionData(spec.partitionType());
+      partition.set(0, 3);
+      try (ManifestWriter<DataFile> writer =
+          ManifestFiles.write(version, spec, Files.localOutput(out), 7L)) {
+        writer.add(DataFiles.builder(spec)
+            .withPath("s3://bucket/table/data/a.parquet")
+            .withFormat(FileFormat.PARQUET)
+            .withPartition(partition)
+            .withRecordCount(10)
+            .withFileSizeInBytes(100)
+            .build());
+      }
+    }
+  }
+}
+```
diff --git a/crates/iceberg/testdata/manifests/java-v1-nested-bucket.avro b/crates/iceberg/testdata/manifests/java-v1-nested-bucket.avro
new file mode 100644
index 0000000000000000000000000000000000000000..bcdef51464d50be94afb4f0e761ccfe39c0d5f8d
GIT binary patch
literal 6017
zcmb_g%WvF77;l4+5QJVpoPsp6goKK>_>tG{nMSOXheXOLimb7BHZkk9U3<JGRJrj#
z@CU$|Bj<AI0VGsV5j_HDICAHk8GB~dYdf1wq*bKY^SyrGV}75H{4eg_ThD`YGR4nd
zduq^&lo$ger&$qD!!*R3UN|xamT8Pb5{+_WaPrh(Pi)hO@syn>GZM4o8F^fU8G(0W
z9OcB+^o8-6acK%cM+z1>$=c->{i%82N=4vLyZI<##!oNd_a2^Q$paDA*OoMpks!&K
ziZY-nj>9p@sRwao7w}v&Tpr0OrbP~~O3Xrz06!=2V$oJ-W1Qya3H7*}Lg2)Rizotl
z?%k<rjK~-l5j6(vd1;YrvNcU3OtCi(BO(pA#*jIUeluY&#q?ZiVCiP*0T+@0vbmoG
zn1)G=KL2cpo`m!q?HwZy8MCS)v@J{XjO!gI*%Y%%P=%yx-UHi6oIwnl0b=PeQIZAh
z4@z<NxFAekx1z@nCuqo=lQ!Dpx2<LbSkZZ3IuFB<=NG|*+pFb-C37()VL*J6o%Mus
zWp0_a2f8ZVbwKF!ay}YI!%UQ&7bFEQo)R=(5LGe`iz(Pd$ry>lISOM$&jChJ1iCd>
zQ_a(nhhGsdj6MH?60KC{3_MCGj<`0&jx+Vuab5SOrrVE_VB&Qv*JZaf*&vCEY0QO{
zM^!J#T;eRltnETdrZ8~i^Ln37Y<tI>+4gQ#GGaQFshJR_qeQjdpCGBZHN#P%dP?-<
zT;A+mp2;XV3j-WI5_58jQ<y7Nst$0<@+2Ze;Rw+Lehgg>EHu<8%qQScd#78wb)Q6W
z+zwQIfXcB}fvR73OW|4;la_Bs6#Z&Y@m?^Y3WKN^!R$b68X_Ez&>=qT#!h#VNqIb}
zw+NtPGXR^1N_VX}`V70W)*NT0TH{D@<xqoSqO@0<t<UZl$F~lay9;2s{S~UsU1)-?
zYIFN5f>q5`s4K)#lcQC1aA&c(`tXR7Cvc<qfH|<EYT(>pra9ZJ&URLS&e?8n)P||<
z?5u<xj~DTHJ3(^v!C$25?SjOnnY+p~bM!$`YsLnU+||Yzk9Qf6^lFDG;!O#~jOHy=
zk|CUv_szpN$SzXO)IbhZatN+xzcVkcK8^A;3aOWj$2p<PS8sJgl88)!BMYADy6Q4K
zW@;(+f$%Ba?c>PRX;GeJ;88Y$dTAoe6*=Q%^%%|qb%e8M_(^Bfn{_V6rtG_X9|)2W
z3GRV+5vF8A5Y*m`WH}JcHQSJ+vzMnNfLPb8jgkwNtHeJaNQ$!D?gZOZwHs7>gj$|p
z`%=3Ti<UL@{Kl2fZ4^p$e(&OUb0w^{E`5nmqPo?^X!!lE?S7VufK$zKqi9xloeiUM
z*S6Gt$t}p;VmTXoOYR!Vo&rcF915!0+A;tsgcf!LrF1@FJlU|QR{6HnhFjYelm<O-
zRa=Thu6uV-MmA+JHf{~em}=c3l$9-M1qM{Z{?KJcY1x%pYMX=-pMz8;ig9(*fI%r{
zmA%9Nfpu3b2K*i<Ux5vAEet9=#am^9VGetpfO$#YuVU!S7`R@=P?dq@VgN;qcQ(Y}
zK=Iv94BdoW24$;iZ{pt9jLd=FZ;K4t^rn$5Fl8Br3uRFb<_1!H%_v+bq?RaLNZrk&
zxRS!kxE$4eHMV-qn4Im_m>j^aipdXS%s-s*2R&uv%FfZrrE*`jeT_#d%}ce6)7~<!
z)-tQRuhTy1zN@Y>s0W?lwYYCc2gHc!q<YGXy20wKuEma=j!~Gi&u%q3Cm+CwDfj&9
zgi<({(FWQl-P=?nqR#M?wz({pvsqv;&-8%1!EZy})SQ}qcNacQw|$qR8IE&CA|M#?
z-S5QSK82w3q{sroXFpuG+J_8_eZIR>eDiPb`S;IGw%&jB?Hl|5{k%2{{J--Lw}1HU
zkE8nszZ~{{{dl<l)A0ABp*ehc^zx%m&HLs-@2|Jg^*7gxx3T|D|HoG^Ui|s&&g=W@
H3(Ecn5Q4wY

literal 0
HcmV?d00001

diff --git a/crates/iceberg/testdata/manifests/java-v2-nested-bucket.avro b/crates/iceberg/testdata/manifests/java-v2-nested-bucket.avro
new file mode 100644
index 0000000000000000000000000000000000000000..32a5d0f644954c2e067888875f010a332ac4702a
GIT binary patch
literal 7261
zcmcIpPjB2r6z`u4f>0qqoN^fXREoD<+i8+rsV8Veq&7kQb3j$(&f2q$iPttBJ0(>8
z6np`M_zWDlaX|<nBv4TgJ;4nL@fmn;#-5q=+MCU0kye$uJAQwD@AuxkpR+&u?BZK%
zJUC<%`s3wiR&kOsYiRKzFN4CeE%{3?3aw$^w#E^QLv9TpJ+tr|$F>qW!Sn2xC3rk%
zpO;b2;I}c2IkOFWQS!ukZp#N<^<c?a-u>KcJdFrEWeDQwb|0tc{NOo!-lNBPdR?aV
z??5@oS&-&vqDp8&lW5F%;X_{e34Cuju8#167A1$DD$F8*fH)`d!ffla3C;LnTKFPN
z`M@J9DdQNP7ttNqR>;P*j0<as-&Y=mCC9V`2(u($eo{_G0=u(X^9*a_C}uuCx5*8|
zWJn9@qg{neC&;<m3T%)jg{0FIvSAtk(>|0xa*~coamWaI8j}7kbZZmmR+jRph|+|F
zEM^7cHgRu3el(7Xllr^bs|43GqZfc=v^Z4JV=o|gp*|vjJ83|LgNOHa$<wGfBs=$s
z;1g!7%kT921}ehtIL#*%+oZ~?VR;8qA@U4C-V6zH)h1~k;6EgzIn*AXH@%qTfD?tt
zUis+8-s!iaj~(fM;ogoye^drX!e4`8$n;shMgaptKIzHeD&nKH>!v9)bU^6fd5!BQ
zb?gL7Ns2OynJs~(WJEa&2~-tJuCgRNV?hHB*8p8MwI8IVXw3?B(QdWZJ<&x-CV7~X
zNIJoSHB8SoPV^=7qr@Mb0MQiWvY%@Q57L6hf~ADRaWr2aH=!^&7^HDINqju6t=7{*
z6_n;V#laL6Yyu-$eczm9M~<`QA3M%=osFE|Reg>a+L-DyBP?gjK=YW!rEcmvOY`)$
zC}*5Li2@owkT=PMW-xhcH15(2v6dN<C?rJ+A4EuigKZn}BZ#Pb7WTKBG06h22C4x-
z<@T3>YS0Wz<61G!fXbE3Uk)n4XB(Il#AOJhiQs)fXcCe=x;Gy?Ge~^O7e!qmfUbi8
zwojJ%TyxDqb}H9gcPXw3q=a$cpqvWbOmoZup3r3FV0qgBmN!^}ZQe{1%wwB3SQ0GU
zFtA^NS45zu#-r@uxn%Rq;Sr}#;aV91=CD831Lqtw&D}(vZ7l(vyV>2SEtA>ZS_(Un
zFA@391<Ac>L{nzjYC*zj<}EYLTysz~nz02WZ@F<M@?8WZlkG4?{0S>4ZV59?vSe`T
zzPX(Q`AH_$HF%EH&yi7_j^@_IGp7;HV0-k_@tCt>arV|XB<!IRSZAwc)6<vXeKcFp
z5o~?hcK0~)%+;n|X+cn&AtEVJrUTm40o|_m<RG2Ek`-~dkHw`$c0WQdb7C`T%+nmu
z%0u9Co7}0lN3l)chqJKT!ddV!*Tsf8jdM0;Im)2WpVb@nSgAL6#fBZY@lJ?H64DAu
zT(9z1XLtGcaqdyMi{dT_n<p&@g^RWl7O*f@G0YDe3JyE2zWedwdRK_hAPre?1wf5y
z!Pe#C-#cbG2l;ZzwbYv1!!;M^bE&^x?bwxm{rZO5`W6%HAmM94T8me$*g^7Z!Hz}3
z$uuqV`IL;+k4}qyskW<!wL+hjnF|@%P%#S)wWJmO#=+!^4!}%)7F1C^<GqKcgVJKP
z-IyhNZiV7im8GxCDWIujYeyUnu{lr1Y7hDikMN)b>|#E478JDFpDvSPKXg~Sx6{ki
zHxQZ;qT1**dj<1XAWL;px&<17REJ>Fuf8`VW#$W1?L;$Dwn;;pF5;?Fs%kIWsu>$1
zRokCJQ`HD%pl!h&EU!UlOZ*4k=g9UDD@OGb_>ddNutq}us$C#jFjFN!3rb3z!$9RA
z>}v6&9W0gu7(z+Yl7kDKxtbj2Ydw{eqenrTtUF_7F0_4DX3(MM&1^=5sWd#GMRh^7
z=&dtm;Q^64vhbjE=g;C)b)@ogZ>qc+Lg<V+xtpCixqw}r(<n-)cxx@b=xHaXc8;lA
zbojdM8xloZUZ`c<?w0Y4mRUZ0Q&Fh!J$;J-A58Jk5x%7YkRxWUiqLNK<<%4n9e&hw
zj3bV(b*C3|xHhUUV8m3fkm?iK;8aCh6on2?bFd*Ew6XRgwnhg;)X~WyeoAoj)rE?4
zLyxkikmv}sx<h2GrUiwzYF@&v+M&yMV{iJuGz6#u_^hk7M=br{UWC82b-hV1a++}5
z$bcU5P5mQh^8jeX(=rd3xJ@Bztvui_5}?zsR2%DSxCmeP^wr-FU;Xp_&0p@VfA#k8
z<u{j(gVA5X<Hy<6&F_AHxqt2MuY0}UZtULqdH0X~U3>S%{)-Pjwy)WDdw;%>t-Zch
UzDY;#41W0O!sR<_Yct0F1BK~hQvd(}

literal 0
HcmV?d00001


From b07024bbde2a1b9b5fd0a5a0ba7b72128bdf67fb Mon Sep 17 00:00:00 2001
From: Ethan Woodbury <woodburyethan60@gmail.com>
Date: Wed, 28 Oct 2026 01:41:03 -0700
Subject: [PATCH 2/2] test(python): expect the partition value matched by field
 ID

---
 bindings/python/tests/test_manifest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/python/tests/test_manifest.py b/bindings/python/tests/test_manifest.py
index 147b81fccc..b095befd45 100644
--- a/bindings/python/tests/test_manifest.py
+++ b/bindings/python/tests/test_manifest.py
@@ -146,7 +146,7 @@ def test_read_manifest_entry(generated_manifest_entry_file: str) -> None:
         == "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet"
     )
     assert data_file.file_format == FileFormat.PARQUET
-    assert repr(data_file.partition) == "[1, None]"
+    assert repr(data_file.partition) == "[1, 1925]"
     assert data_file.record_count == 19513
     assert data_file.file_size_in_bytes == 388872
     assert data_file.column_sizes == {