fix: coerce polars.Array into a suitable Arrow list type

Ideally polars wouldn't be giving us `item` as the list field name, but
it's more important to Just Work ™️ than be pedantic about these
things

Fixes #3566

Signed-off-by: R. Tyler Croy <[email protected]>

Author: rtyler

crates/core/src/kernel/schema/cast/mod.rs

@@ -2,8 +2,8 @@
 //!
 use arrow_array::cast::AsArray;
 use arrow_array::{
-    new_null_array, Array, ArrayRef, GenericListArray, MapArray, OffsetSizeTrait, RecordBatch,
-    RecordBatchOptions, StructArray,
+    new_null_array, Array, ArrayRef, FixedSizeListArray, GenericListArray, MapArray,
+    OffsetSizeTrait, RecordBatch, RecordBatchOptions, StructArray,
 };
 use arrow_cast::{cast_with_options, CastOptions};
 use arrow_schema::{ArrowError, DataType, FieldRef, Fields, SchemaRef as ArrowSchemaRef};
@@ -104,6 +104,24 @@ fn cast_field(
                 add_missing,
             )?) as ArrayRef)
         }
+        (DataType::FixedSizeList(_, _), DataType::FixedSizeList(child_fields, _)) => {
+            let to_type =
+                DataType::new_list(child_fields.data_type().clone(), child_fields.is_nullable());
+            let col = arrow::compute::kernels::cast(
+                col.as_any()
+                    .downcast_ref::<FixedSizeListArray>()
+                    .ok_or_else(|| {
+                        ArrowError::CastError(format!(
+                            "Failed to convert a FixedSizeList into a new list {} ({col_type})",
+                            field.name()
+                        ))
+                    })?,
+                &to_type,
+            )?;
+            // Once the FixedSizeList has been converted to a regular list, go through the usual
+            // list casting code
+            cast_field(&col, field, cast_options, add_missing)
+        }
         (DataType::List(_), DataType::List(child_fields)) => Ok(Arc::new(cast_list(
             col.as_any()
                 .downcast_ref::<GenericListArray<i32>>()

python/deltalake/writer/_conversion.py

@@ -60,6 +60,15 @@ def list_to_delta_dtype(
         if DataType.is_large_list(dtype):
             return DataType.large_list(inner_field_casted)
         elif DataType.is_fixed_size_list(dtype):
+            # Fixed sized lists can come in from polars via their Array type.
+            # These may carry array field names of "item" rather than "element"
+            # which is expected everywhere else. Converting the field name and
+            # then passing the field through for further casting in Rust will
+            # accommodate this
+            #
+            # See also: <https://github.com/delta-io/delta-rs/issues/3566>
+            if inner_field_casted.name == "item":
+                inner_field_casted = inner_field_casted.with_name("element")
             return DataType.list(inner_field_casted, dtype.list_size)
         elif DataType.is_large_list_view(dtype):
             return DataType.large_list_view(inner_field_casted)

python/tests/test_writer.py

@@ -2478,3 +2478,28 @@ def test_write_binary_col_with_dssc(tmp_path: pathlib.Path):
     assert stats["null_count.z.z"].to_pylist() == [1]
     assert stats["min.z.z"].to_pylist() == [101]
     assert stats["max.z.z"].to_pylist() == [104]
+
+
+@pytest.mark.polars
+@pytest.mark.pyarrow
+def test_polars_write_array(tmp_path: pathlib.Path):
+    """
+    https://github.com/delta-io/delta-rs/issues/3566
+    """
+    import polars as pl
+
+    from deltalake import DeltaTable, write_deltalake
+
+    df = pl.DataFrame(
+        {"array": [[-5, -4], [0, 0], [5, 9]]}, schema={"array": pl.Array(pl.Int32, 2)}
+    )
+    DeltaTable.create(
+        tmp_path,
+        df.to_arrow().schema,
+        mode="overwrite",
+    )
+    write_deltalake(
+        tmp_path,
+        df,
+        mode="overwrite",
+    )