From 5f593da9abde43183d932a443a3b26b4d93704ad Mon Sep 17 00:00:00 2001 From: Mark Farnan Date: Fri, 30 Dec 2022 15:11:25 +0100 Subject: [PATCH] Merged AVRO-3631 - Fix for serialization of FIXED [u8,16] - req for uuid. Squashed commit of the following: commit 3a1fac3f0c260b7ec2164701e726234dac6faa6b Author: Martin Tzvetanov Grigorov Date: Fri Oct 7 10:48:13 2022 +0300 AVRO-3631: Use official serde_bytes crate Signed-off-by: Martin Tzvetanov Grigorov commit f7f0ec56f6e39d6fcaffb52586d02e5522c9472e Author: Martin Tzvetanov Grigorov Date: Thu Oct 6 14:01:19 2022 +0300 AVRO-3631: Fix clippy issues Signed-off-by: Martin Tzvetanov Grigorov commit 0ee5b00e7fcccb153fc1b662b503f5dd2ee7ed24 Author: Martin Tzvetanov Grigorov Date: Thu Oct 6 13:38:10 2022 +0300 AVRO-3631: Add more test cases Signed-off-by: Martin Tzvetanov Grigorov commit 70d6cf0dbb5d729633c9c090d4a9198e64e9a0bd Author: Martin Tzvetanov Grigorov Date: Thu Oct 6 12:05:08 2022 +0300 AVRO-3631: Fix clippy and Rat issues Signed-off-by: Martin Tzvetanov Grigorov commit 7c9e938eb67f1812fb3be1e341987003ef2f359f Author: Martin Tzvetanov Grigorov Date: Thu Oct 6 11:45:50 2022 +0300 AVRO-3631: Add serde serialize_with functions Those should be used for hinting the serialization process how to serialize a byte array to Value::(Bytes|Fixed) Signed-off-by: Martin Tzvetanov Grigorov commit 3e3f125ebfd1526eb8ba627e1e6f72c9137d11b7 Author: Martin Tzvetanov Grigorov Date: Mon Oct 3 23:45:11 2022 +0300 AVRO-3631: Use #[serde(with)] attribute to get rid of implementation detail ByteArray Signed-off-by: Martin Tzvetanov Grigorov commit 7b1cdd8e5a5e153db713d03b432e4784ad9fcfe6 Author: Martin Tzvetanov Grigorov Date: Mon Oct 3 14:52:24 2022 +0300 AVRO-3631: Add support for ser_de Value::Fixed It is based on https://github.com/serde-rs/bytes/pull/28 which is not yet merged. Signed-off-by: Martin Tzvetanov Grigorov commit 5c2d1972d8c6a1ae56df6d163c33d54bfa4f8b88 Author: Rik Heijdens Date: Thu Sep 29 14:15:49 2022 +0200 AVRO-3631: Reformat using rustfmt commit a31fcfc96493e3180490dec5622cab485bf7cd79 Author: Rik Heijdens Date: Thu Sep 29 14:14:35 2022 +0200 AVRO-3631: Add test for serializing fixed fields This test-case mainly demonstrates the issue reported in AVRO-3631. It is unclear to me whether we should actually expect the serializer to serialize to a Value::Fixed right away given that Schema information is not available at this stage. commit 12ef14b6a5cc102bcc0317251cd37471148d4926 Author: Rik Heijdens Date: Thu Sep 29 14:12:51 2022 +0200 AVRO-3651: Add test to de.rs to illustrate issue with Fixed fields commit d32d43678d7b6f8eedc08c573be9093824abb146 Author: Rik Heijdens Date: Wed Sep 28 15:48:29 2022 +0200 AVRO-3631: Add test-case to reproduce --- lang/rust/Cargo.lock | 9 + lang/rust/avro/Cargo.toml | 2 + lang/rust/avro/src/de.rs | 184 +++++++++++++++++++- lang/rust/avro/src/lib.rs | 2 +- lang/rust/avro/src/ser.rs | 140 +++++++++++++++- lang/rust/avro/src/types.rs | 326 ++++++------------------------------ lang/rust/avro/tests/ser.rs | 53 ++++++ 7 files changed, 432 insertions(+), 284 deletions(-) create mode 100644 lang/rust/avro/tests/ser.rs diff --git a/lang/rust/Cargo.lock b/lang/rust/Cargo.lock index 05c8ee85290..7c7864fc7e3 100644 --- a/lang/rust/Cargo.lock +++ b/lang/rust/Cargo.lock @@ -65,8 +65,10 @@ dependencies = [ "pretty_assertions", "quad-rand", "rand", + "ref_thread_local", "regex", "serde", + "serde_bytes", "serde_json", "sha2", "snap", @@ -862,6 +864,13 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.7" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.151" diff --git a/lang/rust/avro/Cargo.toml b/lang/rust/avro/Cargo.toml index 09da5801987..cc640d56a34 100644 --- a/lang/rust/avro/Cargo.toml +++ b/lang/rust/avro/Cargo.toml @@ -64,6 +64,7 @@ log = { default-features = false, version = "0.4.17" } num-bigint = { default-features = false, version = "0.4.3" } regex = { default-features = false, version = "1.7.0", features = ["std", "perf"] } serde = { default-features = false, version = "1.0.151", features = ["derive"] } +serde_bytes = { path = "../../../../../external/bytes" } serde_json = { default-features = false, version = "1.0.91", features = ["std"] } snap = { default-features = false, version = "1.1.0", optional = true } strum = { default-features = false, version = "0.24.1" } @@ -74,6 +75,7 @@ uuid = { default-features = false, version = "1.2.2", features = ["serde", "std" xz2 = { default-features = false, version = "0.1.7", optional = true } zerocopy = { default-features = false, version = "0.6.1" } zstd = { default-features = false, version = "0.12.1+zstd.1.5.2", optional = true } +ref_thread_local = { default-features = false, version = "0.1.1" } [target.'cfg(target_arch = "wasm32")'.dependencies] quad-rand = { default-features = false, version = "0.2.1" } diff --git a/lang/rust/avro/src/de.rs b/lang/rust/avro/src/de.rs index 9204ed146ec..fad57269f3d 100644 --- a/lang/rust/avro/src/de.rs +++ b/lang/rust/avro/src/de.rs @@ -21,6 +21,8 @@ use serde::{ de::{self, DeserializeSeed, Visitor}, forward_to_deserialize_any, Deserialize, }; +use std::fmt::Formatter; +use std::marker::PhantomData; use std::{ collections::{ hash_map::{Keys, Values}, @@ -327,10 +329,28 @@ impl<'a, 'de> de::Deserializer<'de> for &'a Deserializer<'de> { V: Visitor<'de>, { match *self.input { + Value::Array(ref values) => { + let bytes: Result, Self::Error> = values + .iter() + .map(|v| match v { + Value::Int(i) if *i <= (u8::MAX as i32) && *i >= (u8::MIN as i32) => { + Ok(*i as u8) + } + _ => Err(de::Error::custom(format!( + "Byte array can be created only from Value::Int values which could be casted to u8: {:?}", + v + ))), + }) + .collect(); + visitor.visit_byte_buf(bytes?) + } Value::String(ref s) => visitor.visit_bytes(s.as_bytes()), Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => visitor.visit_bytes(bytes), Value::Uuid(ref u) => visitor.visit_bytes(u.as_bytes()), - _ => Err(de::Error::custom("not a string|bytes|fixed")), + _ => Err(de::Error::custom(format!( + "not a array|string|bytes|fixed|uuid: {:?}", + *self.input + ))), } } @@ -400,11 +420,24 @@ impl<'a, 'de> de::Deserializer<'de> for &'a Deserializer<'de> { { match *self.input { Value::Array(ref items) => visitor.visit_seq(SeqDeserializer::new(items)), + Value::Fixed(_len, ref items) => { + let mut arr = [0_u8; 6]; + arr.clone_from_slice(items); + visitor.visit_bytes(&arr) + } Value::Union(_i, ref inner) => match **inner { Value::Array(ref items) => visitor.visit_seq(SeqDeserializer::new(items)), - _ => Err(de::Error::custom("not an array")), + Value::Fixed(_len, ref items) => { + let mut arr = [0_u8; 6]; + arr.clone_from_slice(items); + visitor.visit_bytes(&arr) + } + _ => Err(de::Error::custom(format!("not an array: {:?}", **inner))), }, - _ => Err(de::Error::custom("not an array")), + _ => Err(de::Error::custom(format!( + "not an array: {:?}", + *self.input + ))), } } @@ -597,6 +630,65 @@ pub fn from_value<'de, D: Deserialize<'de>>(value: &'de Value) -> Result(deserializer: D) -> Result<&'a [u8], D::Error> +// where +// D: de::Deserializer<'de>, +// { +// struct BytesVisitor<'a> { +// marker: PhantomData<&'a [u8]>, +// } +// +// impl<'de, 'a> Visitor<'de> for BytesVisitor<'a> { +// type Value = &'a [u8]; +// +// fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { +// formatter.write_str("a byte array reference, e.g. &[u8]") +// } +// +// fn visit_bytes(self, v: &[u8]) -> Result +// where +// E: de::Error, +// { +// Ok(v.clone()) +// } +// } +// +// deserializer.deserialize_bytes(BytesVisitor { +// marker: PhantomData, +// }) +// } +// +// #[allow(dead_code)] +// pub fn avro_deserialize_fixed<'de, D, const N: usize>(deserializer: D) -> Result<[u8; N], D::Error> +// where +// D: de::Deserializer<'de>, +// { +// struct FixedVisitor; +// +// impl<'de, const N: usize> Visitor<'de> for FixedVisitor { +// type Value = [u8; N]; +// +// fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { +// formatter.write_str("a byte array with length, e.g. [u8; N]") +// } +// +// fn visit_bytes(self, v: &[u8]) -> Result +// where +// E: de::Error, +// { +// let mut res = [0_u8; N]; +// res.clone_from_slice(v); +// Ok(res) +// } +// } +// +// deserializer.deserialize_bytes(FixedVisitor::) +// } + #[cfg(test)] mod tests { use pretty_assertions::assert_eq; @@ -1126,4 +1218,90 @@ mod tests { assert_eq!(deserialized, reference); Ok(()) } + + #[test] + fn avro_3631_deserialize_value_to_struct_with_byte_types() { + #[derive(Debug, Deserialize, PartialEq)] + struct TestStructFixedField /*<'a>*/ { + // will be serialized as Value::Array> + // array_field: &'a [u8], + // vec_field: Vec, + + // will be serialized as Value::Fixed + // #[serde(deserialize_with = "avro_deserialize_fixed")] + #[serde(with = "serde_bytes")] + fixed_field: [u8; 6], + // #[serde(serialize_with = "avro_serialize_fixed")] + // fixed_field2: &'a [u8], + // #[serde(serialize_with = "avro_serialize_fixed")] + // vec_field2: Vec, + + // will be serialized as Value::Bytes + // #[serde(serialize_with = "avro_serialize_bytes")] + // bytes_field: &'a [u8], + // #[serde(serialize_with = "avro_serialize_bytes")] + // bytes_field2: [u8; 6], + // #[serde(serialize_with = "avro_serialize_bytes")] + // vec_field3: Vec, + } + + let expected = TestStructFixedField { + // array_field: &[1, 11, 111], + // bytes_field: &[2, 22, 222], + // bytes_field2: [2; 6], + fixed_field: [1; 6], + // fixed_field2: &[6, 66], + // vec_field: vec![3, 33], + // vec_field2: vec![4, 44], + // vec_field3: vec![5, 55], + }; + + let value = Value::Record(vec![ + // ( + // "array_field".to_owned(), + // Value::Array( + // expected + // .array_field + // .iter() + // .map(|i| Value::Int(*i as i32)) + // .collect(), + // ), + // ), + // ( + // "vec_field".to_owned(), + // Value::Array( + // expected + // .vec_field + // .iter() + // .map(|i| Value::Int(*i as i32)) + // .collect(), + // ), + // ), + ( + "fixed_field".to_owned(), + Value::Fixed(6, Vec::from(expected.fixed_field)), + ), + // ( + // "fixed_field2".to_owned(), + // Value::Fixed(2, Vec::from(expected.fixed_field2)), + // ), + // ( + // "vec_field2".to_owned(), + // Value::Fixed(2, expected.vec_field2.clone()), + // ), + // ( + // "bytes_field".to_owned(), + // Value::Bytes(Vec::from(expected.bytes_field)), + // ), + // ( + // "bytes_field2".to_owned(), + // Value::Bytes(Vec::from(expected.bytes_field2)), + // ), + // ( + // "vec_field3".to_owned(), + // Value::Bytes(expected.vec_field3.clone()), + // ), + ]); + assert_eq!(expected, from_value(&value).unwrap()); + } } diff --git a/lang/rust/avro/src/lib.rs b/lang/rust/avro/src/lib.rs index 2c6f46c07a3..65ffd4dd2fe 100644 --- a/lang/rust/avro/src/lib.rs +++ b/lang/rust/avro/src/lib.rs @@ -747,7 +747,7 @@ pub use reader::{ SpecificSingleObjectReader, }; pub use schema::{AvroSchema, Schema}; -pub use ser::to_value; +pub use ser::{avro_serialize_bytes, avro_serialize_fixed, to_value}; pub use util::max_allocation_bytes; pub use writer::{ to_avro_datum, to_avro_datum_schemata, GenericSingleObjectWriter, SpecificSingleObjectWriter, diff --git a/lang/rust/avro/src/ser.rs b/lang/rust/avro/src/ser.rs index 7388c366372..7a349c33f95 100644 --- a/lang/rust/avro/src/ser.rs +++ b/lang/rust/avro/src/ser.rs @@ -17,9 +17,24 @@ //! Logic for serde-compatible serialization. use crate::{types::Value, Error}; +use ref_thread_local::{ref_thread_local, RefThreadLocal}; use serde::{ser, Serialize}; use std::{collections::HashMap, iter::once}; +ref_thread_local! { + /// A thread local that is used to decide how to serialize + /// a byte array into Avro `types::Value`. + /// + /// Depends on the fact that serde's serialization process is single-threaded! + static managed BYTES_TYPE: BytesType = BytesType::Bytes; +} + +/// A hint helping in the serialization of a byte arrays (&[u8], [u8; N]) +enum BytesType { + Bytes, + Fixed, +} + #[derive(Clone, Default)] pub struct Serializer {} @@ -174,7 +189,10 @@ impl<'b> ser::Serializer for &'b mut Serializer { } fn serialize_bytes(self, v: &[u8]) -> Result { - Ok(Value::Bytes(v.to_owned())) + match *BYTES_TYPE.borrow() { + BytesType::Bytes => Ok(Value::Bytes(v.to_owned())), + BytesType::Fixed => Ok(Value::Fixed(v.len(), v.to_owned())), + } } fn serialize_none(self) -> Result { @@ -475,7 +493,7 @@ impl<'a> ser::SerializeStructVariant for StructVariantSerializer<'a> { } } -/// Interpret a serializeable instance as a `Value`. +/// Interpret a serializable instance as a `Value`. /// /// This conversion can fail if the value is not valid as per the Avro specification. /// e.g: HashMap with non-string keys @@ -484,6 +502,42 @@ pub fn to_value(value: S) -> Result { value.serialize(&mut serializer) } +/// A function that could be used by #[serde(serialize_with = ...)] to give a +/// hint to Avro's `Serializer` how to serialize a byte array like `[u8; N]` to +/// `Value::Fixed` +#[allow(dead_code)] +pub fn avro_serialize_fixed(value: &[u8], serializer: S) -> Result +where + S: ser::Serializer, +{ + serialize_bytes_type(value, serializer, BytesType::Fixed) +} + +/// A function that could be used by #[serde(serialize_with = ...)] to give a +/// hint to Avro's `Serializer` how to serialize a byte array like `&[u8]` to +/// `Value::Bytes` +#[allow(dead_code)] +pub fn avro_serialize_bytes(value: &[u8], serializer: S) -> Result +where + S: ser::Serializer, +{ + serialize_bytes_type(value, serializer, BytesType::Bytes) +} + +fn serialize_bytes_type( + value: &[u8], + serializer: S, + bytes_type: BytesType, +) -> Result +where + S: ser::Serializer, +{ + *BYTES_TYPE.borrow_mut() = bytes_type; + let res = serializer.serialize_bytes(value); + *BYTES_TYPE.borrow_mut() = BytesType::Bytes; + res +} + #[cfg(test)] mod tests { use super::*; @@ -994,4 +1048,86 @@ mod tests { "error serializing tuple untagged enum" ); } + + #[test] + fn avro_3631_test_to_value_fixed_field() { + #[derive(Debug, Serialize)] + struct TestStructFixedField<'a> { + // will be serialized as Value::Array> + array_field: &'a [u8], + vec_field: Vec, + + // will be serialized as Value::Fixed + #[serde(serialize_with = "avro_serialize_fixed")] + fixed_field: [u8; 6], + #[serde(serialize_with = "avro_serialize_fixed")] + fixed_field2: &'a [u8], + #[serde(serialize_with = "avro_serialize_fixed")] + vec_field2: Vec, + + // will be serialized as Value::Bytes + #[serde(serialize_with = "avro_serialize_bytes")] + bytes_field: &'a [u8], + #[serde(serialize_with = "avro_serialize_bytes")] + bytes_field2: [u8; 6], + #[serde(serialize_with = "avro_serialize_bytes")] + vec_field3: Vec, + } + + let test = TestStructFixedField { + array_field: &[1, 11, 111], + bytes_field: &[2, 22, 222], + bytes_field2: [2; 6], + fixed_field: [1; 6], + fixed_field2: &[6, 66], + vec_field: vec![3, 33], + vec_field2: vec![4, 44], + vec_field3: vec![5, 55], + }; + let expected = Value::Record(vec![ + ( + "array_field".to_owned(), + Value::Array( + test.array_field + .iter() + .map(|i| Value::Int(*i as i32)) + .collect(), + ), + ), + ( + "vec_field".to_owned(), + Value::Array( + test.vec_field + .iter() + .map(|i| Value::Int(*i as i32)) + .collect(), + ), + ), + ( + "fixed_field".to_owned(), + Value::Fixed(6, Vec::from(test.fixed_field)), + ), + ( + "fixed_field2".to_owned(), + Value::Fixed(2, Vec::from(test.fixed_field2)), + ), + ( + "vec_field2".to_owned(), + Value::Fixed(2, test.vec_field2.clone()), + ), + ( + "bytes_field".to_owned(), + Value::Bytes(Vec::from(test.bytes_field)), + ), + ( + "bytes_field2".to_owned(), + Value::Bytes(Vec::from(test.bytes_field2)), + ), + ( + "vec_field3".to_owned(), + Value::Bytes(test.vec_field3.clone()), + ), + ]); + assert_eq!(expected, to_value(test).unwrap()); + } } diff --git a/lang/rust/avro/src/types.rs b/lang/rust/avro/src/types.rs index 228b40ff5ef..5818a407564 100644 --- a/lang/rust/avro/src/types.rs +++ b/lang/rust/avro/src/types.rs @@ -561,11 +561,8 @@ impl Value { } }) } - (_v, _s) => { - println!(":-----------------------------------------------------------------------------------------------------"); - println!("{:?} :-: {:?}", _v, _s); - println!("Unsupported value-schema combination"); - println!(":-----------------------------------------------------------------------------------------------------"); + (v, s) => { + debug!("Unsupported value-schema ({:?}-{:?}) combination:", v, s); Some("Unsupported value-schema combination".to_string()) } } @@ -1036,10 +1033,12 @@ mod tests { decimal::Decimal, duration::{Days, Duration, Millis, Months}, schema::{Name, RecordField, RecordFieldOrder, Schema, UnionSchema}, + to_value, types::Value, }; use apache_avro_test_helper::logger::{assert_logged, assert_not_logged}; use pretty_assertions::assert_eq; + use serde::{Deserialize, Serialize}; use uuid::Uuid; #[test] @@ -2486,282 +2485,53 @@ Field with name '"b"' is not a member of the map items"#, ); } - fn avro_3674_with_or_without_namespace(with_namespace: bool) { - use crate::ser::Serializer; - use serde::Serialize; - - let schema_str = r#"{ - "type": "record", - "name": "NamespacedMessage", - [NAMESPACE] - "fields": [ - { - "type": "record", - "name": "field_a", - "fields": [ - { - "name": "enum_a", - "type": { - "type": "enum", - "name": "EnumType", - "symbols": [ - "SYMBOL_1", - "SYMBOL_2" - ], - "default": "SYMBOL_1" - } - }, - { - "name": "enum_b", - "type": "EnumType" - } - ] - } - ] - }"#; - let schema_str = schema_str.replace( - "[NAMESPACE]", - if with_namespace { - r#""namespace": "com.domain","# - } else { - "" - }, - ); - - let schema = Schema::parse_str(&schema_str).unwrap(); - - #[derive(Serialize)] - enum EnumType { - #[serde(rename = "SYMBOL_1")] - Symbol1, - #[serde(rename = "SYMBOL_2")] - Symbol2, - } - - #[derive(Serialize)] - struct FieldA { - enum_a: EnumType, - enum_b: EnumType, - } - - #[derive(Serialize)] - struct NamespacedMessage { - field_a: FieldA, - } - - let msg = NamespacedMessage { - field_a: FieldA { - enum_a: EnumType::Symbol2, - enum_b: EnumType::Symbol1, - }, - }; - - let mut ser = Serializer::default(); - let test_value: Value = msg.serialize(&mut ser).unwrap(); - assert!(test_value.validate(&schema), "test_value should validate"); - assert!( - test_value.resolve(&schema).is_ok(), - "test_value should resolve" - ); - } - #[test] - fn test_avro_3674_validate_no_namespace_resolution() { - avro_3674_with_or_without_namespace(false); - } - - #[test] - fn test_avro_3674_validate_with_namespace_resolution() { - avro_3674_with_or_without_namespace(true); - } - - fn avro_3688_schema_resolution_panic(set_field_b: bool) { - use crate::ser::Serializer; - use serde::{Deserialize, Serialize}; - - let schema_str = r#"{ - "type": "record", - "name": "Message", - "fields": [ - { - "name": "field_a", - "type": [ - "null", - { - "name": "Inner", - "type": "record", - "fields": [ - { - "name": "inner_a", - "type": "string" - } - ] - } - ], - "default": null - }, - { - "name": "field_b", - "type": [ - "null", - "Inner" - ], - "default": null - } - ] - }"#; - - #[derive(Serialize, Deserialize)] - struct Inner { - inner_a: String, + fn avro_3631_test_serialize_fixed_fields() { + use crate::{avro_serialize_bytes, avro_serialize_fixed}; + + #[derive(Debug, Serialize, Deserialize)] + struct TestStructFixedField<'a> { + #[serde(serialize_with = "avro_serialize_bytes")] + bytes_field: &'a [u8], + #[serde(serialize_with = "avro_serialize_bytes")] + vec_field: Vec, + #[serde(serialize_with = "avro_serialize_fixed")] + fixed_field: [u8; 6], } - #[derive(Serialize, Deserialize)] - struct Message { - field_a: Option, - field_b: Option, - } - - let schema = Schema::parse_str(schema_str).unwrap(); - - let msg = Message { - field_a: Some(Inner { - inner_a: "foo".to_string(), - }), - field_b: if set_field_b { - Some(Inner { - inner_a: "bar".to_string(), - }) - } else { - None - }, - }; - - let mut ser = Serializer::default(); - let test_value: Value = msg.serialize(&mut ser).unwrap(); - assert!(test_value.validate(&schema), "test_value should validate"); - assert!( - test_value.resolve(&schema).is_ok(), - "test_value should resolve" - ); - } - - #[test] - fn test_avro_3688_field_b_not_set() { - avro_3688_schema_resolution_panic(false); - } - - #[test] - fn test_avro_3688_field_b_set() { - avro_3688_schema_resolution_panic(true); - } - - #[test] - fn validate_record_union() { - // { - // "type": "record", - // "fields": [ - // {"type": "long", "name": "a"}, - // {"type": "string", "name": "b"}, - // { - // "type": [ "int", "long"] - // "name": "c", - // "default": null - // } - // ] - // } - let schema = Schema::Record { - name: Name::new("some_record").unwrap(), - aliases: None, - doc: None, - fields: vec![ - RecordField { - name: "a".to_string(), - doc: None, - default: None, - schema: Schema::Long, - order: RecordFieldOrder::Ascending, - position: 0, - custom_attributes: Default::default(), - }, - RecordField { - name: "b".to_string(), - doc: None, - default: None, - schema: Schema::String, - order: RecordFieldOrder::Ascending, - position: 1, - custom_attributes: Default::default(), - }, - RecordField { - name: "c".to_string(), - doc: None, - default: Some(JsonValue::Null), - schema: Schema::Union( - UnionSchema::new(vec![Schema::Int, Schema::Long]).unwrap(), - ), - order: RecordFieldOrder::Ascending, - position: 2, - custom_attributes: Default::default(), - }, - ], - lookup: [ - ("a".to_string(), 0), - ("b".to_string(), 1), - ("c".to_string(), 2), - ] - .iter() - .cloned() - .collect(), - attributes: Default::default(), + let test = TestStructFixedField { + bytes_field: &[1, 2, 3], + fixed_field: [1; 6], + vec_field: vec![2, 3, 4], }; - - // assert!(Value::Map( - // vec![ - // ("a".to_string(), Value::Long(42i64)), - // ("b".to_string(), Value::String("foo".to_string())), - // ( - // "c".to_string(), - // Value::Union(2, Box::new(Value::Long(42i64))) - // ), - // ] - // .into_iter() - // .collect() - // ) - // .validate(&schema)); - - assert!(Value::Record(vec![ - ("a".to_string(), Value::Long(42i64)), - ("b".to_string(), Value::String("foo".to_string())), - ( - "c".to_string(), - Value::Union(2, Box::new(Value::Long(24i64))) - ), - ]) - .validate(&schema)); - - // assert!(Value::Map( - // vec![ - // ("a".to_string(), Value::Long(42i64)), - // ("b".to_string(), Value::String("foo".to_string())), - // ( - // "c".to_string(), - // Value::Union(2, Box::new(Value::Long(42i64))) - // ), - // ] - // .into_iter() - // .collect() - // ) - // .validate(&schema)); - - // assert!(Value::Union( - // 1, - // Box::new(Value::Record(vec![ - // ("a".to_string(), Value::Long(42i64)), - // ("b".to_string(), Value::String("foo".to_string())), - // ])) - // ) - - // assert!(Value::Union(2, Box::new(Value::Long(42i64)),).validate(&union_schema)); + let value: Value = to_value(test).unwrap(); + let schema = Schema::parse_str( + r#" + { + "type": "record", + "name": "TestStructFixedField", + "fields": [ + { + "name": "bytes_field", + "type": "bytes" + }, + { + "name": "vec_field", + "type": "bytes" + }, + { + "name": "fixed_field", + "type": { + "name": "fixed_field", + "type": "fixed", + "size": 6 + } + } + ] + } + "#, + ) + .unwrap(); + assert!(value.validate(&schema)); } } diff --git a/lang/rust/avro/tests/ser.rs b/lang/rust/avro/tests/ser.rs new file mode 100644 index 00000000000..20bc79c1aec --- /dev/null +++ b/lang/rust/avro/tests/ser.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use apache_avro::{to_value, types::Value}; +use serde::{Deserialize, Serialize}; + +#[test] +fn avro_3631_visibility_of_avro_serialize_bytes_type() { + use apache_avro::{avro_serialize_bytes, avro_serialize_fixed}; + + #[derive(Debug, Serialize, Deserialize)] + struct TestStructFixedField<'a> { + // will be serialized as Value::Bytes + #[serde(serialize_with = "avro_serialize_bytes")] + bytes_field: &'a [u8], + + // will be serialized as Value::Fixed + #[serde(serialize_with = "avro_serialize_fixed")] + fixed_field: [u8; 6], + } + + let test = TestStructFixedField { + bytes_field: &[2, 22, 222], + fixed_field: [1; 6], + }; + + let expected = Value::Record(vec![ + ( + "bytes_field".to_owned(), + Value::Bytes(Vec::from(test.bytes_field)), + ), + ( + "fixed_field".to_owned(), + Value::Fixed(6, Vec::from(test.fixed_field)), + ), + ]); + + assert_eq!(expected, to_value(test).unwrap()); +}