Use delta encoding for timestamp (#341)

parseablehq · Mar 13, 2023 · 360b46c · 360b46c
1 parent 30b4353
commit 360b46c
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/server/src/storage/object_storage.rs b/server/src/storage/object_storage.rs
@@ -32,7 +32,10 @@ use crate::{
 use actix_web_prometheus::PrometheusMetrics;
 use async_trait::async_trait;
 use bytes::Bytes;
-use datafusion::arrow::datatypes::Schema;
+use datafusion::{
+    arrow::datatypes::Schema,
+    parquet::{basic::Encoding, schema::types::ColumnPath},
+};
 use datafusion::{
     arrow::{
         array::TimestampMillisecondArray, ipc::reader::StreamReader, record_batch::RecordBatch,
@@ -324,6 +327,10 @@ pub trait ObjectStorage: Sync + 'static {
                 let props = WriterProperties::builder()
                     .set_max_row_group_size(CONFIG.parseable.row_group_size)
                     .set_compression(CONFIG.parseable.parquet_compression.into())
+                    .set_column_encoding(
+                        ColumnPath::new(vec!["p_timestamp".to_string()]),
+                        Encoding::DELTA_BINARY_PACKED,
+                    )
                     .build();
                 let schema = Arc::new(record_reader.merged_schema());
                 let mut writer = ArrowWriter::try_new(parquet_file, schema.clone(), Some(props))?;