Skip to content
This repository has been archived by the owner on Aug 13, 2024. It is now read-only.

feat: Ser/de for schema #95

Merged
merged 2 commits into from
Jul 18, 2023
Merged

feat: Ser/de for schema #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7966478
feat: Ser/de for schema
liurenjie1024 Jul 18, 2023
379ebc2
format
liurenjie1024 Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 104 additions & 75 deletions src/types/on_disk/schema.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,26 @@
use serde::{Deserialize, Serialize};

use super::types::*;
use crate::types;
use crate::Error;
use crate::Result;
use crate::{types, ErrorKind};

/// Parse schema from json bytes.
pub fn parse_schema(schema: &[u8]) -> Result<types::Schema> {
let schema: Schema = serde_json::from_slice(schema)?;
schema.try_into()
}

#[derive(Serialize, Deserialize)]
/// Serialize schema to json string.
pub fn serialize_schema(schema: &types::Schema) -> Result<String> {
Ok(serde_json::to_string(&Schema::try_from(schema)?)?)
}

#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
pub struct Schema {
schema_id: i32,
#[serde(skip_serializing_if = "Option::is_none")]
identifier_field_ids: Option<Vec<i32>>,
fields: Vec<Field>,
}
Expand All @@ -39,21 +45,38 @@ impl TryFrom<Schema> for types::Schema {
impl<'a> TryFrom<&'a types::Schema> for Schema {
type Error = Error;

fn try_from(_value: &'a types::Schema) -> Result<Self> {
Err(Error::new(
ErrorKind::IcebergFeatureUnsupported,
"Serializing schema!",
))
fn try_from(v: &'a types::Schema) -> Result<Self> {
Ok(Self {
schema_id: v.schema_id,
identifier_field_ids: v.identifier_field_ids.as_ref().cloned(),
fields: v
.fields
.iter()
.map(|v| Field::try_from(v.clone()))
.collect::<Result<Vec<Field>>>()?,
})
}
}

#[cfg(test)]
mod tests {
use super::*;

fn check_schema_serde(json_schema: &str, expected_schema: types::Schema) {
let schema = parse_schema(json_schema.as_bytes()).unwrap();
assert_eq!(expected_schema, schema);

let serialized_json_schema = serialize_schema(&expected_schema).unwrap();

assert_eq!(
expected_schema,
parse_schema(serialized_json_schema.as_bytes()).unwrap()
);
}

#[test]
fn test_parse_schema_struct() {
let schema = r#"
fn test_schema_json_conversion() {
let json_schema = r#"
{
"type" : "struct",
"schema-id" : 0,
Expand All @@ -66,18 +89,21 @@ mod tests {
}
"#;

let schema = parse_schema(schema.as_bytes()).unwrap();

assert_eq!(schema.schema_id, 0);
assert_eq!(schema.identifier_field_ids, None);
assert_eq!(schema.fields.len(), 1);
assert_eq!(schema.fields[0].id, 1);
assert_eq!(schema.fields[0].name, "VendorID");
assert!(!schema.fields[0].required);
assert_eq!(
schema.fields[0].field_type,
types::Any::Primitive(types::Primitive::Long)
);
let expected_schema = types::Schema {
schema_id: 0,
identifier_field_ids: None,
fields: vec![types::Field {
id: 1,
name: "VendorID".to_string(),
required: false,
field_type: types::Any::Primitive(types::Primitive::Long),
comment: None,
initial_default: None,
write_default: None,
}],
};

check_schema_serde(json_schema, expected_schema);
}

#[test]
Expand All @@ -97,31 +123,28 @@ mod tests {
}
"#;

let expected_schema = types::Schema {
schema_id: 0,
identifier_field_ids: None,
fields: vec![types::Field {
id: 1,
name: "VendorID".to_string(),
required: false,
field_type: types::Any::Primitive(types::Primitive::Long),
comment: None,
initial_default: Some(types::AnyValue::Primitive(types::PrimitiveValue::Long(123))),
write_default: Some(types::AnyValue::Primitive(types::PrimitiveValue::Long(456))),
}],
};

let schema = parse_schema(schema.as_bytes()).unwrap();

assert_eq!(schema.schema_id, 0);
assert_eq!(schema.identifier_field_ids, None);
assert_eq!(schema.fields.len(), 1);
assert_eq!(schema.fields[0].id, 1);
assert_eq!(schema.fields[0].name, "VendorID");
assert!(!schema.fields[0].required);
assert_eq!(
schema.fields[0].field_type,
types::Any::Primitive(types::Primitive::Long)
);
assert_eq!(
schema.fields[0].initial_default,
Some(types::AnyValue::Primitive(types::PrimitiveValue::Long(123)))
);
assert_eq!(
schema.fields[0].write_default,
Some(types::AnyValue::Primitive(types::PrimitiveValue::Long(456)))
);
assert_eq!(expected_schema, schema);
}

#[test]
fn test_parse_schema_list() {
let schema = r#"
let json_schema = r#"
{
"type" : "struct",
"schema-id" : 0,
Expand All @@ -141,27 +164,30 @@ mod tests {
}
"#;

let schema = parse_schema(schema.as_bytes()).unwrap();

assert_eq!(schema.schema_id, 0);
assert_eq!(schema.identifier_field_ids, None);
assert_eq!(schema.fields.len(), 1);
assert_eq!(schema.fields[0].id, 1);
assert_eq!(schema.fields[0].name, "VendorID");
assert!(!schema.fields[0].required);
assert_eq!(
schema.fields[0].field_type,
types::Any::List(types::List {
element_id: 3,
element_required: true,
element_type: types::Any::Primitive(types::Primitive::String).into(),
})
);
let expected_schema = types::Schema {
schema_id: 0,
identifier_field_ids: None,
fields: vec![types::Field {
id: 1,
name: "VendorID".to_string(),
required: false,
field_type: types::Any::List(types::List {
element_id: 3,
element_required: true,
element_type: types::Any::Primitive(types::Primitive::String).into(),
}),
comment: None,
initial_default: None,
write_default: None,
}],
};

check_schema_serde(json_schema, expected_schema);
}

#[test]
fn test_parse_schema_map() {
let schema = r#"
let json_schema = r#"
{
"type" : "struct",
"schema-id" : 0,
Expand All @@ -183,23 +209,26 @@ mod tests {
}
"#;

let schema = parse_schema(schema.as_bytes()).unwrap();

assert_eq!(schema.schema_id, 0);
assert_eq!(schema.identifier_field_ids, None);
assert_eq!(schema.fields.len(), 1);
assert_eq!(schema.fields[0].id, 1);
assert_eq!(schema.fields[0].name, "VendorID");
assert!(!schema.fields[0].required);
assert_eq!(
schema.fields[0].field_type,
types::Any::Map(types::Map {
key_id: 4,
key_type: types::Any::Primitive(types::Primitive::String).into(),
value_id: 5,
value_required: false,
value_type: types::Any::Primitive(types::Primitive::Double).into(),
})
);
let expected_schema = types::Schema {
schema_id: 0,
identifier_field_ids: None,
fields: vec![types::Field {
id: 1,
name: "VendorID".to_string(),
required: false,
field_type: types::Any::Map(types::Map {
key_id: 4,
key_type: types::Any::Primitive(types::Primitive::String).into(),
value_id: 5,
value_required: false,
value_type: types::Any::Primitive(types::Primitive::Double).into(),
}),
comment: None,
initial_default: None,
write_default: None,
}],
};

check_schema_serde(json_schema, expected_schema);
}
}
Loading