Skip to content

Commit

Permalink
models: add sourceCapture to materialization specs
Browse files Browse the repository at this point in the history
Adds the `sourceCapture` property to materialization specs. This is an optional
field that can specify a capture from which new bindings will be added. As
bindings are added to the capture, they should also be added to the
materialization. This allows materializations of dynamic sets of source
collections based on the set of resources that's discovered in an upstream data
source.

The `sourceCapture` field is not used directly during build/validation. This
is because we want to decouple the builds that modify the materializations from
those that modify the capture bindings. In other words, the intent is to use a
separate publication for modifying each materialization in response to capture
bindings modifications.
  • Loading branch information
psFried committed Aug 7, 2023
1 parent f63cf2c commit 02a6af2
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 1 deletion.
8 changes: 7 additions & 1 deletion crates/models/src/materializations.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
use super::{is_false, ConnectorConfig, Field, RawValue, RelativeUrl, ShardTemplate, Source};
use super::{
is_false, Capture, ConnectorConfig, Field, RawValue, RelativeUrl, ShardTemplate, Source,
};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::json;
Expand All @@ -9,6 +11,9 @@ use std::collections::BTreeMap;
#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct MaterializationDef {
/// # Automatically materialize new bindings from a named capture
#[serde(skip_serializing_if = "Option::is_none")]
pub source_capture: Option<Capture>,
/// # Endpoint to materialize into.
pub endpoint: MaterializationEndpoint,
/// # Bound collections to materialize into the endpoint.
Expand Down Expand Up @@ -85,6 +90,7 @@ pub struct MaterializationFields {
impl MaterializationDef {
pub fn example() -> Self {
Self {
source_capture: None,
endpoint: MaterializationEndpoint::Connector(ConnectorConfig::example()),
bindings: vec![MaterializationBinding::example()],
shards: ShardTemplate::default(),
Expand Down
1 change: 1 addition & 0 deletions crates/sources/src/inline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ fn inline_materialization(
materialization: _,
spec:
models::MaterializationDef {
source_capture: _,
endpoint,
bindings,
shards: _,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -752,6 +752,10 @@ expression: "&schema"
"shards": {
"title": "Template for shards of this materialization task.",
"$ref": "#/definitions/ShardTemplate"
},
"sourceCapture": {
"title": "Automatically materialize new bindings from a named capture",
"$ref": "#/definitions/Capture"
}
},
"additionalProperties": false
Expand Down
4 changes: 4 additions & 0 deletions flow.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,10 @@
"shards": {
"title": "Template for shards of this materialization task.",
"$ref": "#/definitions/ShardTemplate"
},
"sourceCapture": {
"title": "Automatically materialize new bindings from a named capture",
"$ref": "#/definitions/Capture"
}
},
"additionalProperties": false
Expand Down

0 comments on commit 02a6af2

Please sign in to comment.