From a58f3436ed4eb86f4db210cddbf2cdba6432b65d Mon Sep 17 00:00:00 2001 From: Marcin Kuthan Date: Mon, 23 Oct 2023 15:08:29 +0200 Subject: [PATCH] flex templates - 1 (#119) --- .../actions/build-flex-template/action.yml | 35 ++++++ .github/actions/setup/action.yml | 11 +- .../{automerge.yml => auto-merge.yml} | 6 +- .github/workflows/ci.yml | 22 +--- .github/workflows/deploy.yml | 33 ------ .github/workflows/flex-templates.yml | 47 ++++++++ .github/workflows/scala-steward.yml | 2 +- .github/workflows/tf-apply.yml | 26 +++++ .github/workflows/tf-plan.yml | 28 +++-- README.md | 3 +- toll-application/deploy-batch.sh | 11 -- toll-application/deploy-streaming.sh | 11 -- toll-application/run-batch.sh | 12 ++- toll-application/run-streaming.sh | 7 +- toll-infrastructure/main.tf | 101 +++++++++++++++--- toll-infrastructure/schemas/diagnostic.json | 18 ++++ .../schemas/io-diagnostic.json | 10 -- .../schemas/toll-booth-diagnostic.json | 12 ++- .../schemas/toll-booth-entry-stats.json | 22 +++- .../schemas/toll-booth-entry.json | 34 +++++- .../schemas/toll-booth-exit.json | 8 +- .../schemas/total-vehicle-times.json | 20 +++- .../schemas/vehicle-registration.json | 8 +- .../vehicles-with-expired-registration.json | 16 ++- 24 files changed, 364 insertions(+), 139 deletions(-) create mode 100644 .github/actions/build-flex-template/action.yml rename .github/workflows/{automerge.yml => auto-merge.yml} (67%) delete mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/flex-templates.yml create mode 100644 .github/workflows/tf-apply.yml delete mode 100755 toll-application/deploy-batch.sh delete mode 100755 toll-application/deploy-streaming.sh create mode 100644 toll-infrastructure/schemas/diagnostic.json delete mode 100644 toll-infrastructure/schemas/io-diagnostic.json diff --git a/.github/actions/build-flex-template/action.yml b/.github/actions/build-flex-template/action.yml new file mode 100644 index 00000000..1c7609b4 --- /dev/null +++ b/.github/actions/build-flex-template/action.yml @@ -0,0 +1,35 @@ +name: Build flex template +description: Build flex template using gcloud + +inputs: + template_file: + description: Google Cloud Storage location of the flex template file + required: true + image_gcr: + description: Google Artifact Registry location to store the flex template image to be built + required: true + artifact_name: + description: Artifact name + required: true + main_class: + description: Main class + required: true + +runs: + using: composite + steps: + - uses: actions/download-artifact@v3 + with: + name: ${{ inputs.artifact_name }} + + - uses: "google-github-actions/setup-gcloud@v1" + + - name: Flex template build + shell: bash + run: | + gcloud dataflow flex-template build ${{ inputs.template_file }} \ + --image-gcr-path "${{ image_gcr }}" \ + --sdk-language "JAVA" \ + --flex-template-base-image JAVA17 \ + --jar "${{ inputs.artifact_name }}" \ + --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="${{ inputs.main_class }}" \ diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 35850ae6..d9cd01ca 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,12 +1,11 @@ -name: 'Setup' -description: 'Setup build' +name: Setup +description: Setup java runs: - using: "composite" + using: composite steps: - - name: Setup JDK - uses: actions/setup-java@v3 + - uses: actions/setup-java@v3 with: distribution: temurin java-version: 17 - cache: sbt \ No newline at end of file + cache: sbt diff --git a/.github/workflows/automerge.yml b/.github/workflows/auto-merge.yml similarity index 67% rename from .github/workflows/automerge.yml rename to .github/workflows/auto-merge.yml index 99f0f790..2f467ea3 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/auto-merge.yml @@ -1,4 +1,4 @@ -name: Scala Steward auto-merge +name: Auto-merge on: pull_request @@ -10,10 +10,10 @@ jobs: automerge: runs-on: ubuntu-latest - if: ${{ github.actor == 'mkuthan-scala-steward[bot]' }} + if: ${{ github.actor == 'mkuthan-scala-steward[bot]' || github.actor == 'dependabot[bot]' }} steps: - - name: Enable auto-merge for Scala Steward PRs + - name: Enable auto-merge run: gh pr merge --auto --squash "$PR_URL" env: PR_URL: ${{github.event.pull_request.html_url}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bf962eb..777f9072 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,8 @@ name: CI -on: [push, workflow_dispatch] +on: + workflow_dispatch: ~ + pull_request: ~ concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -32,25 +34,9 @@ jobs: - uses: ./.github/actions/setup - run: sbt "scalafixAll --check" - tf-validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dflook/terraform-validate@v1 - with: - path: toll-infrastructure - - tf-fmt: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dflook/terraform-fmt-check@v1 - with: - path: toll-infrastructure - test: runs-on: ubuntu-latest - needs: [scalafmt, scapegoat, scalafix, tf-validate, tf-fmt] + needs: [scalafmt, scapegoat, scalafix] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 9d31c71c..00000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Deploy - -on: - workflow_dispatch: ~ - workflow_run: - workflows: ["CI"] - types: [completed] - branches: - - "main" - -jobs: - deploy: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup - - run: sbt tollApplication/assembly - - uses: google-github-actions/auth@v1 - with: - credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" - - uses: dflook/terraform-apply@v1 - with: - path: toll-infrastructure - # TODO: fix IAM and deploy flex templates - # - uses: "google-github-actions/setup-gcloud@v1" - # - run: ./deploy-batch.sh - # working-directory: toll-application - # - run: ./deploy-streaming.sh - # working-directory: toll-application - - # TODO: run streaming job - # TODO: deploy DAG for batch job diff --git a/.github/workflows/flex-templates.yml b/.github/workflows/flex-templates.yml new file mode 100644 index 00000000..f4e4bbd1 --- /dev/null +++ b/.github/workflows/flex-templates.yml @@ -0,0 +1,47 @@ +name: Build flex templates + +on: + workflow_dispatch: ~ + push: + branches: + - main + +jobs: + assembly: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - run: sbt tollApplication/assembly + - uses: actions/upload-artifact@v3 + with: + name: toll-application + path: toll-application/target/scala-2.13/toll-application.jar" + + build-batch: + runs-on: ubuntu-latest + needs: [assembly] + steps: + - uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" + - uses: ./.github/actions/build-flex-template + with: + template_file: gs://playground-272019-toll-application/templates/toll-application-batch.json + image_gcr: europe-west1-docker.pkg.dev/playground-272019/toll-application/toll-application-batch:latest + artifact_name: toll-application + main_class: org.mkuthan.streamprocessing.toll.application.batch.TollBatchJob + + build-streaming: + runs-on: ubuntu-latest + needs: [assembly] + steps: + - uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" + - uses: ./.github/actions/build-flex-template + with: + template_file: gs://playground-272019-toll-application/templates/toll-application-streaming.json + image_gcr: europe-west1-docker.pkg.dev/playground-272019/toll-application/toll-application-streaming:latest + artifact_name: toll-application + main_class: org.mkuthan.streamprocessing.toll.application.streaming.TollStreamingJob diff --git a/.github/workflows/scala-steward.yml b/.github/workflows/scala-steward.yml index 4a6e0380..58a8c4ca 100644 --- a/.github/workflows/scala-steward.yml +++ b/.github/workflows/scala-steward.yml @@ -1,9 +1,9 @@ name: Scala Steward on: + workflow_dispatch: ~ schedule: - cron: "0 0 * * 0" # 00:00 every Sunday - workflow_dispatch: jobs: scala-steward: diff --git a/.github/workflows/tf-apply.yml b/.github/workflows/tf-apply.yml new file mode 100644 index 00000000..fa51b1f4 --- /dev/null +++ b/.github/workflows/tf-apply.yml @@ -0,0 +1,26 @@ +name: Terraform apply + +on: + workflow_dispatch: ~ + push: + branches: + - main + +permissions: + contents: read + pull-requests: write + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + apply: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" + - uses: dflook/terraform-apply@v1 + with: + path: toll-infrastructure diff --git a/.github/workflows/tf-plan.yml b/.github/workflows/tf-plan.yml index 8caf1cd3..dbafbe4e 100644 --- a/.github/workflows/tf-plan.yml +++ b/.github/workflows/tf-plan.yml @@ -1,25 +1,41 @@ name: Terraform plan -on: [pull_request] +on: + workflow_dispatch: ~ + pull_request: ~ permissions: contents: read pull-requests: write +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: - plan: + validate: runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dflook/terraform-validate@v1 + with: + path: toll-infrastructure - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + fmt: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: dflook/terraform-fmt-check@v1 + with: + path: toll-infrastructure + plan: + runs-on: ubuntu-latest + needs: [validate, fmt] + steps: + - uses: actions/checkout@v4 - uses: google-github-actions/auth@v1 with: credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" - - uses: dflook/terraform-plan@v1 with: path: toll-infrastructure diff --git a/README.md b/README.md index 4bbe009b..25d1ad9f 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,8 @@ Shared modules: Use cases: -* `toll-application`, `toll-domain` - sample application for toll data processing, +* `toll-application`, `toll-domain`, `toll-infrastructure` - sample application for toll data processing, see [blog post](https://mkuthan.github.io/blog/2023/09/27/unified-batch-streaming/) -* `toll-infrastructure` - GCP infrastructure as a code for sample application * `word-count` - fixed window example, see [blog post](https://mkuthan.github.io/blog/2022/01/28/stream-processing-part1/) * `session-window` - session window example, diff --git a/toll-application/deploy-batch.sh b/toll-application/deploy-batch.sh deleted file mode 100755 index 962940f5..00000000 --- a/toll-application/deploy-batch.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -PROJECT=playground-272019 -REGION=europe-west1 - -gcloud dataflow flex-template build gs://$PROJECT-toll-application/templates/toll-application-batch.json \ - --image-gcr-path "$REGION-docker.pkg.dev/$PROJECT/toll-application/toll-application-batch:latest" \ - --sdk-language "JAVA" \ - --flex-template-base-image JAVA17 \ - --jar "target/scala-2.13/toll-application.jar" \ - --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="org.mkuthan.streamprocessing.toll.application.batch.TollBatchJob" \ diff --git a/toll-application/deploy-streaming.sh b/toll-application/deploy-streaming.sh deleted file mode 100755 index 082f0224..00000000 --- a/toll-application/deploy-streaming.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -PROJECT=playground-272019 -REGION=europe-west1 - -gcloud dataflow flex-template build gs://$PROJECT-toll-application/templates/toll-application-streaming.json \ - --image-gcr-path "$REGION-docker.pkg.dev/$PROJECT/toll-application/toll-application-streaming:latest" \ - --sdk-language "JAVA" \ - --flex-template-base-image JAVA17 \ - --jar "target/scala-2.13/toll-application.jar" \ - --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="org.mkuthan.streamprocessing.toll.application.streaming.TollStreamingJob" \ diff --git a/toll-application/run-batch.sh b/toll-application/run-batch.sh index 570341ab..a99f3e49 100755 --- a/toll-application/run-batch.sh +++ b/toll-application/run-batch.sh @@ -1,15 +1,17 @@ #!/bin/bash +DATE=`date +%Y-%m-%d` + PROJECT=playground-272019 REGION=europe-west1 -DATE=`date +%Y-%m-%d` gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \ --template-file-gcs-location "gs://$PROJECT-toll-application/templates/toll-application-batch.json" \ --region "$REGION" \ - --staging-location="gs://$PROJECT-toll-application/staging/" \ - --additional-experiments="use_runner_v2" \ - --max-workers=1 \ + --staging-location "gs://$PROJECT-toll-application/staging/" \ + --additional-experiments "use_runner_v2" \ + --max-workers 1 \ + --worker-machine-type "t2d-standard-1" \ --parameters effectiveDate="$DATE" \ --parameters entryTable="$PROJECT.toll_application.toll-booth-entry" \ --parameters exitTable="$PROJECT.toll_application.toll-booth-exit" \ @@ -18,5 +20,5 @@ gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \ --parameters entryStatsDailyTable="$PROJECT.toll_application.toll-booth-entry-stats-daily" \ --parameters totalVehicleTimesOneHourGapTable="$PROJECT.toll_application.total-vehicle-times-one-hour-gap" \ --parameters totalVehicleTimesOneHourGapDiagnosticTable="$PROJECT.toll_application.total-vehicle-times-one-hour-gap-diagnostic" \ - --parameters vehiclesWithExpiredRegistrationDailyTable="$PROJECT.toll_application.vehicles-with-expired-registration-diagnostic-daily" \ + --parameters vehiclesWithExpiredRegistrationDailyTable="$PROJECT.toll_application.vehicles-with-expired-registration-daily" \ --parameters vehiclesWithExpiredRegistrationDailyDiagnosticTable="$PROJECT.toll_application.vehicles-with-expired-registration-daily-diagnostic" \ diff --git a/toll-application/run-streaming.sh b/toll-application/run-streaming.sh index 7ff132d8..45fca7f0 100755 --- a/toll-application/run-streaming.sh +++ b/toll-application/run-streaming.sh @@ -6,10 +6,11 @@ REGION=europe-west1 gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \ --template-file-gcs-location "gs://$PROJECT-toll-application/templates/toll-application-streaming.json" \ --region "$REGION" \ - --staging-location="gs://$PROJECT-toll-application/staging/" \ + --staging-location "gs://$PROJECT-toll-application/staging/" \ --enable-streaming-engine \ - --additional-experiments="use_runner_v2" \ - --max-workers=1 \ + --additional-experiments "use_runner_v2" \ + --max-workers 1 \ + --worker-machine-type "t2d-standard-1" \ --parameters entrySubscription="projects/$PROJECT/subscriptions/toll-booth-entry" \ --parameters entryDlq="gs://$PROJECT-toll-application/dlq/entry" \ --parameters exitSubscription="projects/$PROJECT/subscriptions/toll-booth-exit" \ diff --git a/toll-infrastructure/main.tf b/toll-infrastructure/main.tf index 30a30e00..90270b37 100644 --- a/toll-infrastructure/main.tf +++ b/toll-infrastructure/main.tf @@ -58,7 +58,12 @@ resource "google_bigquery_table" "toll-booth-entry-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "entry_time" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-entry.json") } @@ -76,7 +81,12 @@ resource "google_bigquery_table" "toll-booth-exit-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "exit_time" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-exit.json") } @@ -94,7 +104,11 @@ resource "google_bigquery_table" "vehicle-registration-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + require_partition_filter = true + } + schema = file("${path.module}/schemas/vehicle-registration.json") } @@ -103,7 +117,12 @@ resource "google_bigquery_table" "toll-booth-entry-stats-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-entry-stats.json") } @@ -112,7 +131,12 @@ resource "google_bigquery_table" "toll-booth-entry-stats-hourly-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-entry-stats.json") } @@ -121,7 +145,12 @@ resource "google_bigquery_table" "toll-booth-entry-stats-daily-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-entry-stats.json") } @@ -130,7 +159,12 @@ resource "google_bigquery_table" "total-vehicle-times-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/total-vehicle-times.json") } @@ -139,7 +173,12 @@ resource "google_bigquery_table" "total-vehicle-times-diagnostic-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-diagnostic.json") } @@ -148,7 +187,12 @@ resource "google_bigquery_table" "total-vehicle-times-one-hour-gap-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/total-vehicle-times.json") } @@ -157,7 +201,12 @@ resource "google_bigquery_table" "total-vehicle-times-one-hour-gap-diagnostic-ta dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-diagnostic.json") } @@ -170,7 +219,12 @@ resource "google_bigquery_table" "vehicles-with-expired-registration-diagnostic- dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-diagnostic.json") } @@ -179,16 +233,26 @@ resource "google_bigquery_table" "vehicles-with-expired-registration-daily-table dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/vehicles-with-expired-registration.json") } resource "google_bigquery_table" "vehicles-with-expired-registration-daily-diagnostic-table" { - table_id = "vehicles-with-expired-registration-diagnostic-daily" + table_id = "vehicles-with-expired-registration-daily-diagnostic" dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + schema = file("${path.module}/schemas/toll-booth-diagnostic.json") } @@ -197,6 +261,11 @@ resource "google_bigquery_table" "io-diagnostic-table" { dataset_id = google_bigquery_dataset.toll-application-dataset.dataset_id deletion_protection = false - # TODO: define schema - schema = file("${path.module}/schemas/io-diagnostic.json") + time_partitioning { + type = "DAY" + field = "created_at" + require_partition_filter = true + } + + schema = file("${path.module}/schemas/diagnostic.json") } diff --git a/toll-infrastructure/schemas/diagnostic.json b/toll-infrastructure/schemas/diagnostic.json new file mode 100644 index 00000000..b51e5cab --- /dev/null +++ b/toll-infrastructure/schemas/diagnostic.json @@ -0,0 +1,18 @@ +[ + { + "name": "created_at", + "type": "TIMESTAMP" + }, + { + "name": "id", + "type": "STRING" + }, + { + "name": "reason", + "type": "STRING" + }, + { + "name": "count", + "type": "INTEGER" + } +] \ No newline at end of file diff --git a/toll-infrastructure/schemas/io-diagnostic.json b/toll-infrastructure/schemas/io-diagnostic.json deleted file mode 100644 index d4bd31a0..00000000 --- a/toll-infrastructure/schemas/io-diagnostic.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "name": "permalink", - "type": "STRING" - }, - { - "name": "state", - "type": "STRING" - } -] \ No newline at end of file diff --git a/toll-infrastructure/schemas/toll-booth-diagnostic.json b/toll-infrastructure/schemas/toll-booth-diagnostic.json index d4bd31a0..b51e5cab 100644 --- a/toll-infrastructure/schemas/toll-booth-diagnostic.json +++ b/toll-infrastructure/schemas/toll-booth-diagnostic.json @@ -1,10 +1,18 @@ [ { - "name": "permalink", + "name": "created_at", + "type": "TIMESTAMP" + }, + { + "name": "id", "type": "STRING" }, { - "name": "state", + "name": "reason", "type": "STRING" + }, + { + "name": "count", + "type": "INTEGER" } ] \ No newline at end of file diff --git a/toll-infrastructure/schemas/toll-booth-entry-stats.json b/toll-infrastructure/schemas/toll-booth-entry-stats.json index d4bd31a0..fc2b34f7 100644 --- a/toll-infrastructure/schemas/toll-booth-entry-stats.json +++ b/toll-infrastructure/schemas/toll-booth-entry-stats.json @@ -1,10 +1,26 @@ [ { - "name": "permalink", - "type": "STRING" + "name": "created_at", + "type": "TIMESTAMP" }, { - "name": "state", + "name": "id", "type": "STRING" + }, + { + "name": "count", + "type": "INTEGER" + }, + { + "name": "total_toll", + "type": "NUMERIC" + }, + { + "name": "first_entry_time", + "type": "TIMESTAMP" + }, + { + "name": "last_entry_time", + "type": "TIMESTAMP" } ] \ No newline at end of file diff --git a/toll-infrastructure/schemas/toll-booth-entry.json b/toll-infrastructure/schemas/toll-booth-entry.json index d4bd31a0..25e5fbb7 100644 --- a/toll-infrastructure/schemas/toll-booth-entry.json +++ b/toll-infrastructure/schemas/toll-booth-entry.json @@ -1,10 +1,42 @@ [ { - "name": "permalink", + "name": "id", + "type": "STRING" + }, + { + "name": "entry_time", + "type": "TIMESTAMP" + }, + { + "name": "license_plate", "type": "STRING" }, { "name": "state", "type": "STRING" + }, + { + "name": "make", + "type": "STRING" + }, + { + "name": "model", + "type": "STRING" + }, + { + "name": "vehicle_type", + "type": "STRING" + }, + { + "name": "weight_type", + "type": "STRING" + }, + { + "name": "toll", + "type": "NUMERIC" + }, + { + "name": "tag", + "type": "STRING" } ] \ No newline at end of file diff --git a/toll-infrastructure/schemas/toll-booth-exit.json b/toll-infrastructure/schemas/toll-booth-exit.json index d4bd31a0..2721c6b2 100644 --- a/toll-infrastructure/schemas/toll-booth-exit.json +++ b/toll-infrastructure/schemas/toll-booth-exit.json @@ -1,10 +1,14 @@ [ { - "name": "permalink", + "name": "id", "type": "STRING" }, { - "name": "state", + "name": "exit_time", + "type": "TIMESTAMP" + }, + { + "name": "license_plate", "type": "STRING" } ] \ No newline at end of file diff --git a/toll-infrastructure/schemas/total-vehicle-times.json b/toll-infrastructure/schemas/total-vehicle-times.json index d4bd31a0..74543009 100644 --- a/toll-infrastructure/schemas/total-vehicle-times.json +++ b/toll-infrastructure/schemas/total-vehicle-times.json @@ -1,10 +1,26 @@ [ { - "name": "permalink", + "name": "created_at", + "type": "TIMESTAMP" + }, + { + "name": "license_plate", "type": "STRING" }, { - "name": "state", + "name": "toll_booth_id", "type": "STRING" + }, + { + "name": "entry_time", + "type": "TIMESTAMP" + }, + { + "name": "exit_time", + "type": "TIMESTAMP" + }, + { + "name": "duration_seconds", + "type": "INTEGER" } ] \ No newline at end of file diff --git a/toll-infrastructure/schemas/vehicle-registration.json b/toll-infrastructure/schemas/vehicle-registration.json index d4bd31a0..715ba150 100644 --- a/toll-infrastructure/schemas/vehicle-registration.json +++ b/toll-infrastructure/schemas/vehicle-registration.json @@ -1,10 +1,14 @@ [ { - "name": "permalink", + "name": "id", "type": "STRING" }, { - "name": "state", + "name": "license_plate", "type": "STRING" + }, + { + "name": "expired", + "type": "INTEGER" } ] \ No newline at end of file diff --git a/toll-infrastructure/schemas/vehicles-with-expired-registration.json b/toll-infrastructure/schemas/vehicles-with-expired-registration.json index d4bd31a0..84dba6c0 100644 --- a/toll-infrastructure/schemas/vehicles-with-expired-registration.json +++ b/toll-infrastructure/schemas/vehicles-with-expired-registration.json @@ -1,10 +1,22 @@ [ { - "name": "permalink", + "name": "created_at", + "type": "TIMESTAMP" + }, + { + "name": "vehicle_registration_id", "type": "STRING" }, { - "name": "state", + "name": "toll_booth_id", + "type": "STRING" + }, + { + "name": "entry_time", + "type": "TIMESTAMP" + }, + { + "name": "license_plate", "type": "STRING" } ] \ No newline at end of file