Skip to content

Commit

Permalink
assemble: set default flush_interval to 24h
Browse files Browse the repository at this point in the history
  • Loading branch information
@mdibaiee
mdibaiee committed Oct 23, 2024
1 parent f0eb24f commit 5650ead
Show file tree
Hide file tree
Showing 12 changed files with 246 additions and 45 deletions.
1 change: 1 addition & 0 deletions crates/activate/src/snapshots/activate__test__ops_collection_partition.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ expression: "json!([\"list_req\", list_req, \"spec\", spec])"
"flags": 4,
"fragment": {
"compressionCodec": "GZIP",
"flushInterval": "86400s",
"length": "536870912",
"pathPostfixTemplate": "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
"refreshInterval": "300s",
Expand Down
6 changes: 3 additions & 3 deletions crates/assemble/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ pub fn partition_template(
// Use a supplied compression codec. Or, if none, then default to gzip.
let compression_codec = compression_codec(codec.unwrap_or(models::CompressionCodec::Gzip));

// If an explicit flush interval isn't provided, then don't set one.
let flush_interval = flush_interval.map(Into::into);
// If an explicit flush interval isn't provided, default to 24 hours
let flush_interval = flush_interval.unwrap_or(std::time::Duration::from_secs(24 * 3600)).into();

// If a fragment length isn't set, default and then map MB to bytes.
let length = (length.unwrap_or(512) as i64) << 20;
Expand Down Expand Up @@ -156,7 +156,7 @@ pub fn partition_template(
replication,
fragment: Some(broker::journal_spec::Fragment {
compression_codec: compression_codec as i32,
flush_interval,
flush_interval: Some(flush_interval),
length,
path_postfix_template,
refresh_interval,
Expand Down
7 changes: 6 additions & 1 deletion .../validation/tests/snapshots/scenario_tests__collection_schema_contains_flow_document.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down
21 changes: 18 additions & 3 deletions ...s/snapshots/scenario_tests__connector_validation_is_skipped_when_shards_are_disabled.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down Expand Up @@ -564,7 +569,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down Expand Up @@ -835,7 +845,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down
42 changes: 36 additions & 6 deletions crates/validation/tests/snapshots/scenario_tests__disabled_bindings_are_ignored.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down Expand Up @@ -666,7 +671,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down Expand Up @@ -876,7 +886,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down Expand Up @@ -1211,7 +1226,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down Expand Up @@ -1381,7 +1401,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down Expand Up @@ -1915,7 +1940,12 @@ Outcome {
},
),
retention: None,
flush_interval: None,
flush_interval: Some(
Duration {
seconds: 86400,
nanos: 0,
},
),
path_postfix_template: "utc_date={{.Spool.FirstAppendTime.Format \"2006-01-02\"}}/utc_hour={{.Spool.FirstAppendTime.Format \"15\"}}",
},
),
Expand Down
Loading

0 comments on commit 5650ead

Please sign in to comment.