feat: Reassign field ids for schema (#615)

* Reassign field ids for schema * Address comments * Schema ensure unique field ids * Fix tests with duplicate nested field ids * Use Schema::builder() for reassigned ids * Better docs
apache · Oct 4, 2024 · 2bc66c4 · 2bc66c4
1 parent d3b3ab1
commit 2bc66c4
Show file tree

Hide file tree

Showing 4 changed files with 411 additions and 32 deletions.
diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs
@@ -826,8 +826,8 @@ mod tests {
 
     fn arrow_schema_for_arrow_schema_to_schema_test() -> ArrowSchema {
         let fields = Fields::from(vec![
-            simple_field("key", DataType::Int32, false, "17"),
-            simple_field("value", DataType::Utf8, true, "18"),
+            simple_field("key", DataType::Int32, false, "28"),
+            simple_field("value", DataType::Utf8, true, "29"),
         ]);
 
         let r#struct = DataType::Struct(fields);
@@ -1057,9 +1057,9 @@ mod tests {
                     "required": true,
                     "type": {
                         "type": "map",
-                        "key-id": 17,
+                        "key-id": 28,
                         "key": "int",
-                        "value-id": 18,
+                        "value-id": 29,
                         "value-required": false,
                         "value": "string"
                     }
@@ -1110,8 +1110,8 @@ mod tests {
 
     fn arrow_schema_for_schema_to_arrow_schema_test() -> ArrowSchema {
         let fields = Fields::from(vec![
-            simple_field("key", DataType::Int32, false, "17"),
-            simple_field("value", DataType::Utf8, true, "18"),
+            simple_field("key", DataType::Int32, false, "28"),
+            simple_field("value", DataType::Utf8, true, "29"),
         ]);
 
         let r#struct = DataType::Struct(fields);
@@ -1200,7 +1200,7 @@ mod tests {
             ),
             simple_field("map", map, false, "16"),
             simple_field("struct", r#struct, false, "17"),
-            simple_field("uuid", DataType::FixedSizeBinary(16), false, "26"),
+            simple_field("uuid", DataType::FixedSizeBinary(16), false, "30"),
         ])
     }
 
@@ -1344,9 +1344,9 @@ mod tests {
                     "required": true,
                     "type": {
                         "type": "map",
-                        "key-id": 17,
+                        "key-id": 28,
                         "key": "int",
-                        "value-id": 18,
+                        "value-id": 29,
                         "value-required": false,
                         "value": "string"
                     }
@@ -1380,7 +1380,7 @@ mod tests {
                     }
                 },
                 {
-                    "id":26,
+                    "id":30,
                     "name":"uuid",
                     "required":true,
                     "type":"uuid"

diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs
@@ -668,6 +668,12 @@ impl NestedField {
         self.write_default = Some(value);
         self
     }
+
+    /// Set the id of the field.
+    pub(crate) fn with_id(mut self, id: i32) -> Self {
+        self.id = id;
+        self
+    }
 }
 
 impl fmt::Display for NestedField {