From f58facab287cb37fc0fdc165839edfc819503ae2 Mon Sep 17 00:00:00 2001 From: Jan Kaul Date: Mon, 18 Nov 2024 16:51:37 +0100 Subject: [PATCH] update iceberg-rust --- Cargo.lock | 164 ++++++++++++++++++++++++-------------- frostbow-glue/src/main.rs | 11 +-- frostbow-sql/src/main.rs | 14 +--- 3 files changed, 114 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed74f6c..d64dd85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,7 +169,7 @@ dependencies = [ "snap", "strum 0.25.0", "strum_macros 0.25.3", - "thiserror", + "thiserror 1.0.69", "typed-builder", "uuid", "xz2", @@ -599,9 +599,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53dcf5e7d9bd1517b8b998e170e650047cea8a2b85fe1835abe3210713e541b7" +checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" dependencies = [ "aws-credential-types", "aws-runtime", @@ -889,9 +889,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5327f6c99920069d1fe374aa743be1af0031dea9f250852cdf1ae6a0861ee24" +checksum = "2506947f73ad44e344215ccd6403ac2ae18cd8e046e581a441bf8d199f257f03" dependencies = [ "borsh-derive", "cfg_aliases 0.2.1", @@ -899,9 +899,9 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10aedd8f1a81a8aafbfde924b0e3061cd6fedd6f6bbcfc6a76e6fd426d7bfe26" +checksum = "c2593a3b8b938bd68373196c9832f516be11fa487ef4ae745eb282e6a56a7244" dependencies = [ "once_cell", "proc-macro-crate", @@ -1010,9 +1010,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "jobserver", "libc", @@ -1073,9 +1073,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -1083,9 +1083,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstream", "anstyle", @@ -1107,9 +1107,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "clipboard-win" @@ -1128,13 +1128,13 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "comfy-table" -version = "7.1.1" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" +checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" dependencies = [ "strum 0.26.3", "strum_macros 0.26.4", - "unicode-width", + "unicode-width 0.2.0", ] [[package]] @@ -1830,7 +1830,7 @@ dependencies = [ [[package]] name = "datafusion_iceberg" version = "0.5.8" -source = "git+https://github.com/jankaul/iceberg-rust#7932b699375e8107b9657458639fd0290a7f6b9b" +source = "git+https://github.com/jankaul/iceberg-rust#47c424fdb5ccd3dc3a16f06057db02a1547f6f1d" dependencies = [ "async-trait", "chrono", @@ -1843,7 +1843,7 @@ dependencies = [ "object_store", "regex", "serde_json", - "thiserror", + "thiserror 1.0.69", "tokio", "url", "uuid", @@ -2088,9 +2088,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "miniz_oxide", @@ -2315,8 +2315,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -2609,7 +2611,7 @@ dependencies = [ "http 1.1.0", "hyper 1.5.0", "hyper-util", - "rustls 0.23.16", + "rustls 0.23.17", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", @@ -2675,7 +2677,7 @@ dependencies = [ [[package]] name = "iceberg-glue-catalog" version = "0.5.8" -source = "git+https://github.com/jankaul/iceberg-rust#7932b699375e8107b9657458639fd0290a7f6b9b" +source = "git+https://github.com/jankaul/iceberg-rust#47c424fdb5ccd3dc3a16f06057db02a1547f6f1d" dependencies = [ "async-trait", "aws-config", @@ -2684,7 +2686,7 @@ dependencies = [ "iceberg-rust", "object_store", "serde_json", - "thiserror", + "thiserror 1.0.69", "url", "uuid", ] @@ -2692,7 +2694,7 @@ dependencies = [ [[package]] name = "iceberg-rest-catalog" version = "0.5.8" -source = "git+https://github.com/jankaul/iceberg-rust#7932b699375e8107b9657458639fd0290a7f6b9b" +source = "git+https://github.com/jankaul/iceberg-rust#47c424fdb5ccd3dc3a16f06057db02a1547f6f1d" dependencies = [ "async-trait", "futures", @@ -2713,7 +2715,7 @@ dependencies = [ [[package]] name = "iceberg-rust" version = "0.5.8" -source = "git+https://github.com/jankaul/iceberg-rust#7932b699375e8107b9657458639fd0290a7f6b9b" +source = "git+https://github.com/jankaul/iceberg-rust#47c424fdb5ccd3dc3a16f06057db02a1547f6f1d" dependencies = [ "apache-avro", "arrow", @@ -2731,7 +2733,7 @@ dependencies = [ "serde_json", "smallvec", "sqlparser", - "thiserror", + "thiserror 1.0.69", "thrift", "tokio", "url", @@ -2741,7 +2743,7 @@ dependencies = [ [[package]] name = "iceberg-rust-spec" version = "0.5.8" -source = "git+https://github.com/jankaul/iceberg-rust#7932b699375e8107b9657458639fd0290a7f6b9b" +source = "git+https://github.com/jankaul/iceberg-rust#47c424fdb5ccd3dc3a16f06057db02a1547f6f1d" dependencies = [ "apache-avro", "arrow-schema", @@ -2758,7 +2760,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_repr", - "thiserror", + "thiserror 1.0.69", "url", "uuid", ] @@ -2766,7 +2768,7 @@ dependencies = [ [[package]] name = "iceberg-sql-catalog" version = "0.5.8" -source = "git+https://github.com/jankaul/iceberg-rust#7932b699375e8107b9657458639fd0290a7f6b9b" +source = "git+https://github.com/jankaul/iceberg-rust#47c424fdb5ccd3dc3a16f06057db02a1547f6f1d" dependencies = [ "async-trait", "futures", @@ -2774,7 +2776,7 @@ dependencies = [ "object_store", "serde_json", "sqlx", - "thiserror", + "thiserror 1.0.69", "url", "uuid", ] @@ -3093,9 +3095,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.162" +version = "0.2.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libflate" @@ -3515,9 +3517,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.4.0+3.4.0" +version = "300.4.1+3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a709e02f2b4aca747929cca5ed248880847c650233cf8b8cdc48f40aaf4898a6" +checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" dependencies = [ "cc", ] @@ -3840,37 +3842,40 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.16", + "rustls 0.23.17", "socket2", - "thiserror", + "thiserror 2.0.3", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", "rustc-hash", - "rustls 0.23.16", + "rustls 0.23.17", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.3", "tinyvec", "tracing", + "web-time", ] [[package]] @@ -3961,7 +3966,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -4074,7 +4079,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.16", + "rustls 0.23.17", "rustls-native-certs 0.8.0", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -4106,7 +4111,7 @@ dependencies = [ "reqwest 0.11.27", "serde", "task-local-extensions", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -4243,9 +4248,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" dependencies = [ "once_cell", "ring", @@ -4303,6 +4308,9 @@ name = "rustls-pki-types" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -4348,7 +4356,7 @@ dependencies = [ "nix", "radix_trie", "unicode-segmentation", - "unicode-width", + "unicode-width 0.1.14", "utf8parse", "windows-sys 0.52.0", ] @@ -4465,9 +4473,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -4732,14 +4740,14 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rustls 0.23.16", + "rustls 0.23.17", "rustls-pemfile 2.2.0", "serde", "serde_json", "sha2", "smallvec", "sqlformat", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", "tracing", @@ -4823,7 +4831,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 1.0.69", "tracing", "whoami", ] @@ -4861,7 +4869,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 1.0.69", "tracing", "whoami", ] @@ -5074,7 +5082,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +dependencies = [ + "thiserror-impl 2.0.3", ] [[package]] @@ -5088,6 +5105,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "thiserror-impl" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "thrift" version = "0.17.0" @@ -5219,7 +5247,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.16", + "rustls 0.23.17", "rustls-pki-types", "tokio", ] @@ -5400,6 +5428,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -5597,6 +5631,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.6" @@ -6000,9 +6044,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/frostbow-glue/src/main.rs b/frostbow-glue/src/main.rs index f7c7ba4..f6da861 100644 --- a/frostbow-glue/src/main.rs +++ b/frostbow-glue/src/main.rs @@ -16,7 +16,8 @@ use datafusion_iceberg::{ }; use frostbow::{Args, IcebergContext}; use iceberg_glue_catalog::GlueCatalog; -use object_store::{aws::AmazonS3Builder, local::LocalFileSystem, memory::InMemory, ObjectStore}; +use iceberg_rust::catalog::bucket::ObjectStoreBuilder; +use object_store::{aws::AmazonS3Builder, local::LocalFileSystem}; #[tokio::main] async fn main() -> ExitCode { @@ -33,17 +34,17 @@ async fn main_inner() -> Result<(), Error> { let bucket = args.bucket; let command = args.command; - let object_store: Arc = match &bucket { + let object_store: ObjectStoreBuilder = match &bucket { Some(bucket) => { if bucket.starts_with("s3://") { let builder = AmazonS3Builder::from_env().with_bucket_name(bucket); - Arc::new(builder.build()?) + ObjectStoreBuilder::S3(builder) } else { - Arc::new(LocalFileSystem::new()) + ObjectStoreBuilder::Filesystem(Arc::new(LocalFileSystem::new())) } } - _ => Arc::new(InMemory::new()), + _ => ObjectStoreBuilder::memory(), }; let config = aws_config::load_defaults(BehaviorVersion::v2024_03_28()).await; diff --git a/frostbow-sql/src/main.rs b/frostbow-sql/src/main.rs index 39bb37d..09116d8 100644 --- a/frostbow-sql/src/main.rs +++ b/frostbow-sql/src/main.rs @@ -16,7 +16,7 @@ use datafusion_iceberg::{ planner::{IcebergQueryPlanner, RefreshMaterializedView}, }; use frostbow::{Args, IcebergContext}; -use iceberg_rust::catalog::bucket::{Bucket, ObjectStoreBuilder}; +use iceberg_rust::catalog::bucket::ObjectStoreBuilder; use object_store::{aws::AmazonS3Builder, local::LocalFileSystem, memory::InMemory}; use iceberg_sql_catalog::SqlCatalogList; @@ -56,15 +56,9 @@ async fn main_inner() -> Result<(), Error> { let iceberg_catalog_list = { Arc::new( - SqlCatalogList::new( - &catalog_url, - match &bucket { - Some(bucket) => object_store.build(Bucket::S3(&bucket))?, - None => object_store.build(Bucket::Local)?, - }, - ) - .await - .unwrap(), + SqlCatalogList::new(&catalog_url, object_store) + .await + .unwrap(), ) };