diff --git a/Makefile b/Makefile index 80f334a34..13c668d8a 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,9 @@ define spark_jvm_17_extra_args $(shell ./mvnw help:evaluate -Dexpression=extraJavaTestArgs | grep -v '\[') endef +# Build optional Comet native features (like hdfs e.g) +FEATURES_ARG := $(shell ! [ -z $(COMET_FEATURES) ] && echo '--features=$(COMET_FEATURES)') + all: core jvm core: @@ -95,7 +98,7 @@ release-linux: clean cd native && RUSTFLAGS="-Ctarget-cpu=native -Ctarget-feature=-prefer-256-bit" cargo build --release ./mvnw install -Prelease -DskipTests $(PROFILES) release: - cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release + cd native && RUSTFLAGS="$(RUSTFLAGS) -Ctarget-cpu=native" && RUSTFLAGS=$$RUSTFLAGS cargo build --release $(FEATURES_ARG) ./mvnw install -Prelease -DskipTests $(PROFILES) release-nogit: cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release diff --git a/docs/source/user-guide/datasources.md b/docs/source/user-guide/datasources.md index 9607ba603..a152f85c1 100644 --- a/docs/source/user-guide/datasources.md +++ b/docs/source/user-guide/datasources.md @@ -35,3 +35,79 @@ converted into Arrow format, allowing native execution to happen after that. Comet does not provide native JSON scan, but when `spark.comet.convert.json.enabled` is enabled, data is immediately converted into Arrow format, allowing native execution to happen after that. + +# Supported Storages + +## Local +In progress + +## HDFS + +Apache DataFusion Comet native reader seamlessly scans files from remote HDFS for [supported formats](#supported-spark-data-sources) + +### Using experimental native DataFusion reader +Unlike to native Comet reader the Datafusion reader fully supports nested types processing. This reader is currently experimental only + +To build Comet with native DataFusion reader and remote HDFS support it is required to have a JDK installed + +Example: +Build a Comet for `spark-3.4` provide a JDK path in `JAVA_HOME` +Provide the JRE linker path in `RUSTFLAGS`, the path can vary depending on the system. Typically JRE linker is a part of installed JDK + +```shell +export JAVA_HOME="/opt/homebrew/opt/openjdk@11" +make release PROFILES="-Pspark-3.4" COMET_FEATURES=hdfs RUSTFLAGS="-L $JAVA_HOME/libexec/openjdk.jdk/Contents/Home/lib/server" +``` + +Start Comet with experimental reader and HDFS support as [described](installation.md/#run-spark-shell-with-comet-enabled) +and add additional parameters + +```shell +--conf spark.comet.scan.impl=native_datafusion \ +--conf spark.hadoop.fs.defaultFS="hdfs://namenode:9000" \ +--conf spark.hadoop.dfs.client.use.datanode.hostname = true \ +--conf dfs.client.use.datanode.hostname = true +``` + +Query a struct type from Remote HDFS +```shell +spark.read.parquet("hdfs://namenode:9000/user/data").show(false) + +root + |-- id: integer (nullable = true) + |-- first_name: string (nullable = true) + |-- personal_info: struct (nullable = true) + | |-- firstName: string (nullable = true) + | |-- lastName: string (nullable = true) + | |-- ageInYears: integer (nullable = true) + +25/01/30 16:50:43 INFO core/src/lib.rs: Comet native library version 0.6.0 initialized +== Physical Plan == +* CometColumnarToRow (2) ++- CometNativeScan: (1) + + +(1) CometNativeScan: +Output [3]: [id#0, first_name#1, personal_info#4] +Arguments: [id#0, first_name#1, personal_info#4] + +(2) CometColumnarToRow [codegen id : 1] +Input [3]: [id#0, first_name#1, personal_info#4] + + +25/01/30 16:50:44 INFO fs-hdfs-0.1.12/src/hdfs.rs: Connecting to Namenode (hdfs://namenode:9000) ++---+----------+-----------------+ +|id |first_name|personal_info | ++---+----------+-----------------+ +|2 |Jane |{Jane, Smith, 34}| +|1 |John |{John, Doe, 28} | ++---+----------+-----------------+ + + + +``` + +Verify the native scan type should be `CometNativeScan`. + +## S3 +In progress diff --git a/native/Cargo.lock b/native/Cargo.lock index ee734a076..0b8203843 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -399,6 +399,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 1.0.109", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -510,6 +532,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -524,7 +555,9 @@ checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -598,6 +631,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.6", +] + [[package]] name = "clap" version = "4.5.27" @@ -897,6 +941,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions-nested", + "datafusion-objectstore-hdfs", "datafusion-physical-expr", "futures", "hex", @@ -1183,6 +1228,20 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "datafusion-objectstore-hdfs" +version = "0.1.4" +source = "git+https://github.com/comphead/datafusion-objectstore-hdfs?branch=master#411069b0e3bfaad5192e14fb5075d9f66058d39b" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "fs-hdfs", + "futures", + "object_store", + "tokio", +] + [[package]] name = "datafusion-optimizer" version = "44.0.0" @@ -1461,6 +1520,20 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-hdfs" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f164ff6334da016dffd1c29a3c05b81c35b857ef829d3fa9e58ae8d3e6f76b" +dependencies = [ + "bindgen", + "cc", + "lazy_static", + "libc", + "log", + "url", +] + [[package]] name = "futures" version = "0.3.31" @@ -1930,7 +2003,7 @@ dependencies = [ "combine", "java-locator", "jni-sys", - "libloading", + "libloading 0.7.4", "log", "thiserror", "walkdir", @@ -1968,6 +2041,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lexical-core" version = "1.0.5" @@ -2048,6 +2127,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.11" @@ -2172,6 +2261,12 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.3" @@ -2198,6 +2293,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num" version = "0.4.3" @@ -2407,6 +2512,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2754,6 +2865,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.1" @@ -3153,6 +3270,7 @@ checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", + "parking_lot", "pin-project-lite", "tokio-macros", ] diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index d3f17a705..9830f9215 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -77,6 +77,7 @@ datafusion-comet-proto = { workspace = true } object_store = { workspace = true } url = { workspace = true } chrono = { workspace = true } +datafusion-objectstore-hdfs = { git = "https://github.com/comphead/datafusion-objectstore-hdfs", branch = "master", optional = true } [dev-dependencies] pprof = { version = "0.14.0", features = ["flamegraph"] } @@ -88,6 +89,7 @@ hex = "0.4.3" [features] default = [] +hdfs = ["datafusion-objectstore-hdfs"] [lib] name = "comet" diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 3dc59a9fd..51ee5ebb0 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -73,7 +73,7 @@ use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctio use crate::execution::shuffle::CompressionCodec; use crate::execution::spark_plan::SparkPlan; -use crate::parquet::parquet_support::SparkParquetOptions; +use crate::parquet::parquet_support::{register_object_store, SparkParquetOptions}; use crate::parquet::schema_adapter::SparkSchemaAdapterFactory; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::parquet::ParquetExecBuilder; @@ -104,7 +104,6 @@ use datafusion_common::{ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, JoinType as DFJoinType, ScalarValue, }; -use datafusion_execution::object_store::ObjectStoreUrl; use datafusion_expr::type_coercion::other::get_coerce_type_for_case_expression; use datafusion_expr::{ AggregateUDF, ReturnTypeArgs, ScalarUDF, WindowFrame, WindowFrameBound, WindowFrameUnits, @@ -1155,12 +1154,9 @@ impl PhysicalPlanner { )) }); - let object_store = object_store::local::LocalFileSystem::new(); - // register the object store with the runtime environment - let url = Url::try_from("file://").unwrap(); - self.session_ctx - .runtime_env() - .register_object_store(&url, Arc::new(object_store)); + // By default, local FS object store registered + // if `hdfs` feature enabled then HDFS file object store registered + let object_store_url = register_object_store(Arc::clone(&self.session_ctx))?; // Generate file groups let mut file_groups: Vec> = @@ -1219,8 +1215,6 @@ impl PhysicalPlanner { // TODO: I think we can remove partition_count in the future, but leave for testing. assert_eq!(file_groups.len(), partition_count); - - let object_store_url = ObjectStoreUrl::local_filesystem(); let partition_fields: Vec = partition_schema .fields() .iter() diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index 248f2babd..ba7e4ddd1 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::execution::operators::ExecutionError; use arrow::{ array::{ cast::AsArray, @@ -35,11 +36,12 @@ use arrow_array::builder::StringBuilder; use arrow_array::{DictionaryArray, StringArray, StructArray}; use arrow_schema::DataType; use chrono::{NaiveDate, NaiveDateTime, TimeZone, Timelike}; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::prelude::SessionContext; use datafusion_comet_spark_expr::utils::array_with_timezone; use datafusion_comet_spark_expr::{timezone, EvalMode, SparkError, SparkResult}; use datafusion_common::{cast::as_generic_string_array, Result as DataFusionResult, ScalarValue}; use datafusion_expr::ColumnarValue; -// use datafusion_physical_expr::PhysicalExpr; use num::{ cast::AsPrimitive, integer::div_floor, traits::CheckedNeg, CheckedSub, Integer, Num, ToPrimitive, @@ -1861,6 +1863,42 @@ fn trim_end(s: &str) -> &str { } } +// Default object store which is local filesystem +#[cfg(not(feature = "hdfs"))] +pub(crate) fn register_object_store( + session_context: Arc, +) -> Result { + let object_store = object_store::local::LocalFileSystem::new(); + let url = ObjectStoreUrl::parse("file://")?; + session_context + .runtime_env() + .register_object_store(url.as_ref(), Arc::new(object_store)); + Ok(url) +} + +// HDFS object store +#[cfg(feature = "hdfs")] +pub(crate) fn register_object_store( + session_context: Arc, +) -> Result { + // TODO: read the namenode configuration from file schema or from spark.defaultFS + let url = ObjectStoreUrl::parse("hdfs://namenode:9000")?; + if let Some(object_store) = + datafusion_objectstore_hdfs::object_store::hdfs::HadoopFileSystem::new(url.as_ref()) + { + session_context + .runtime_env() + .register_object_store(url.as_ref(), Arc::new(object_store)); + + return Ok(url); + } + + Err(ExecutionError::GeneralError(format!( + "HDFS object store cannot be created for {}", + url + ))) +} + #[cfg(test)] mod tests { use arrow::datatypes::TimestampMicrosecondType;