Skip to content

Commit

Permalink
Fix compatibility changes schema handling apache-avro 0.17
Browse files Browse the repository at this point in the history
- Handle ArraySchema struct
- Handle MapSchema struct
- Map BigDecimal => LargeBinary
- Map TimestampNanos => Timestamp(TimeUnit::Nanosecond, None)
- Map LocalTimestampNanos => todo!()
- Add Default to FixedSchema test
  • Loading branch information
Marc Droogh committed Dec 10, 2024
1 parent ae65b82 commit 55ca31b
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
}
AvroSchema::Array(schema) => {
let sub_parent_field_name = format!("{}.element", parent_field_name);
Self::child_schema_lookup(&sub_parent_field_name, schema, schema_lookup)?;
Self::child_schema_lookup(&sub_parent_field_name, &schema.items, schema_lookup)?;
}
_ => (),
}
Expand Down
8 changes: 6 additions & 2 deletions datafusion/core/src/datasource/avro_to_arrow/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,11 @@ fn schema_to_field_with_props(
AvroSchema::Bytes => DataType::Binary,
AvroSchema::String => DataType::Utf8,
AvroSchema::Array(item_schema) => DataType::List(Arc::new(
schema_to_field_with_props(item_schema, Some("element"), false, None)?,
schema_to_field_with_props(&item_schema.items, Some("element"), false, None)?,
)),
AvroSchema::Map(value_schema) => {
let value_field =
schema_to_field_with_props(value_schema, Some("value"), false, None)?;
schema_to_field_with_props(&value_schema.types, Some("value"), false, None)?;
DataType::Dictionary(
Box::new(DataType::Utf8),
Box::new(value_field.data_type().clone()),
Expand Down Expand Up @@ -144,14 +144,17 @@ fn schema_to_field_with_props(
AvroSchema::Decimal(DecimalSchema {
precision, scale, ..
}) => DataType::Decimal128(*precision as u8, *scale as i8),
AvroSchema::BigDecimal => DataType::LargeBinary,
AvroSchema::Uuid => DataType::FixedSizeBinary(16),
AvroSchema::Date => DataType::Date32,
AvroSchema::TimeMillis => DataType::Time32(TimeUnit::Millisecond),
AvroSchema::TimeMicros => DataType::Time64(TimeUnit::Microsecond),
AvroSchema::TimestampMillis => DataType::Timestamp(TimeUnit::Millisecond, None),
AvroSchema::TimestampMicros => DataType::Timestamp(TimeUnit::Microsecond, None),
AvroSchema::TimestampNanos => DataType::Timestamp(TimeUnit::Nanosecond, None),
AvroSchema::LocalTimestampMillis => todo!(),
AvroSchema::LocalTimestampMicros => todo!(),
AvroSchema::LocalTimestampNanos => todo!(),
AvroSchema::Duration => DataType::Duration(TimeUnit::Millisecond),
};

Expand Down Expand Up @@ -371,6 +374,7 @@ mod test {
aliases: Some(vec![alias("foofixed"), alias("barfixed")]),
size: 1,
doc: None,
default: None,
attributes: Default::default(),
});
let props = external_props(&fixed_schema);
Expand Down

0 comments on commit 55ca31b

Please sign in to comment.