diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 40fb6c72..f90c3887 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,8 +22,8 @@ jobs: - uses: dflook/terraform-apply@v1 with: path: toll-infrastructure - - uses: "google-github-actions/setup-gcloud@v1" - - run: ./deploy-batch.sh - working-directory: toll-application - - run: ./deploy-streaming.sh - working-directory: toll-application + # - uses: "google-github-actions/setup-gcloud@v1" + # - run: ./deploy-batch.sh + # working-directory: toll-application + # - run: ./deploy-streaming.sh + # working-directory: toll-application diff --git a/.gitignore b/.gitignore index 9e79245e..9ef5593b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ metals.sbt .idea .idea_modules /.worksheet/ + +# Terraform +.terraform diff --git a/toll-application/run-batch.sh b/toll-application/run-batch.sh new file mode 100755 index 00000000..5cde41af --- /dev/null +++ b/toll-application/run-batch.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +PROJECT=playground-272019 +REGION=europe-west1 +DATE=`date +%Y-%m-%d` + +gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \ + --template-file-gcs-location "gs://$PROJECT-toll-application/templates/toll-application-batch.json" \ + --region "$REGION" \ + --staging-location="gs://$PROJECT-toll-application/staging/" \ + --parameters effectiveDate="$DATE" \ + --parameters entryTable="$PROJECT.toll_application.toll-booth-entry" \ + --parameters exitTable="$PROJECT.toll_application.toll-booth-exit" \ + --parameters vehicleRegistrationTable="$PROJECT.toll_application.vehicle-registration" \ + --parameters entryStatsHourlyTable="$PROJECT.toll_application.toll-booth-entry-stats-hourly" \ + --parameters entryStatsDailyTable="$PROJECT.toll_application.toll-booth-entry-stats-daily" \ + --parameters totalVehicleTimesOneHourGapTable="$PROJECT.toll_application.total-vehicle-times-one-hour-gap" \ + --parameters totalVehicleTimesOneHourGapDiagnosticTable="$PROJECT.toll_application.total-vehicle-times-one-hour-gap-diagnostic" \ + --parameters vehiclesWithExpiredRegistrationDailyTable="$PROJECT.toll_application.vehicles-with-expired-registration-diagnostic-daily" \ + --parameters vehiclesWithExpiredRegistrationDailyDiagnosticTable="$PROJECT.toll_application.vehicles-with-expired-registration-daily-diagnostic" \ + diff --git a/toll-application/run-streaming.sh b/toll-application/run-streaming.sh index fbeac257..4c70dba7 100755 --- a/toll-application/run-streaming.sh +++ b/toll-application/run-streaming.sh @@ -6,6 +6,7 @@ REGION=europe-west1 gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \ --template-file-gcs-location "gs://$PROJECT-toll-application/templates/toll-application-streaming.json" \ --region "$REGION" \ + --staging-location="gs://$PROJECT-toll-application/staging/" \ --enable-streaming-engine \ --parameters entrySubscription="projects/$PROJECT/subscriptions/toll-booth-entry" \ --parameters entryDlq="gs://$PROJECT-toll-application/dlq/entry" \ @@ -19,4 +20,4 @@ gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \ --parameters totalVehicleTimesDiagnosticTable="$PROJECT.toll_application.total-vehicle-times-diagnostic" \ --parameters vehiclesWithExpiredRegistrationTopic="projects/$PROJECT/topics/vehicle-registration" \ --parameters vehiclesWithExpiredRegistrationDiagnosticTable="$PROJECT.toll_application.vehicles-with-expired-registration-diagnostic" \ - --parameters ioDiagnosticTable="$PROJECT.toll_application.io-diagnostic" + --parameters ioDiagnosticTable="$PROJECT.toll_application.io-diagnostic" \ diff --git a/toll-infrastructure/.terraform.lock.hcl b/toll-infrastructure/.terraform.lock.hcl index bad32cd2..8ae90617 100644 --- a/toll-infrastructure/.terraform.lock.hcl +++ b/toll-infrastructure/.terraform.lock.hcl @@ -20,3 +20,22 @@ provider "registry.terraform.io/hashicorp/google" { "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } + +provider "registry.terraform.io/hashicorp/google-beta" { + version = "5.2.0" + hashes = [ + "h1:Ix3+nRkg9WyF8IamiD6oi1dbhmrLbGgn7+Q+upqzmGY=", + "zh:09834404dd19d9191d29a2a58a6838ecd9f70b54e24e39c75f3063d4345671e5", + "zh:34c8564245834b2f8a2ed7e70880b1553e79dd55083cdadc0791fb2f611cd5a1", + "zh:7ee42223685859efea71bfe90c2b0e37a1ef0b79d523c415fa4196c307dc6024", + "zh:898c2038a828dce2a5acd497a60dd1074cd06f7c45a3b17f3fb7351d61f87f11", + "zh:8d46416318f51f38291724af1c0758abc45424c5f6e76e7405da6284e017e23a", + "zh:a1cb9ddf8f9aca6970efb349cf70fcffa051eb73675137d585a1da857ed2d47f", + "zh:b2f906c9261c7d9ce978f0f2915e26c829a363358d49c788e6cdbb3ebe7965b9", + "zh:b35f0c6167860f2014287336bc52f3c9cd6f5c5e0a1488ac76c2f8dc929077d4", + "zh:b6157769ad44e7e31aed666cdda5cd7e697ec422e0b94c28ba88a393fa5abfbe", + "zh:e15b215803700dae0ff68545f66c56a596e399ad3ecb1a76a924d9bc185b4385", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f9c8fac2a1500b647c9fab69cb5e29c2207c0f1682e497deaf0007ed226e8e4b", + ] +} diff --git a/toll-infrastructure/.terraform/providers/registry.terraform.io/hashicorp/google/5.2.0/darwin_amd64/terraform-provider-google_v5.2.0_x5 b/toll-infrastructure/.terraform/providers/registry.terraform.io/hashicorp/google/5.2.0/darwin_amd64/terraform-provider-google_v5.2.0_x5 deleted file mode 100755 index e6e8e4b5..00000000 Binary files a/toll-infrastructure/.terraform/providers/registry.terraform.io/hashicorp/google/5.2.0/darwin_amd64/terraform-provider-google_v5.2.0_x5 and /dev/null differ diff --git a/toll-infrastructure/.terraform/terraform.tfstate b/toll-infrastructure/.terraform/terraform.tfstate deleted file mode 100644 index a506627e..00000000 --- a/toll-infrastructure/.terraform/terraform.tfstate +++ /dev/null @@ -1,30 +0,0 @@ -{ - "version": 3, - "serial": 1, - "lineage": "8f99eb7c-5ea0-0f52-b5b4-99319448b8b6", - "backend": { - "type": "gcs", - "config": { - "access_token": null, - "bucket": "playground-272019-tf-state", - "credentials": null, - "encryption_key": null, - "impersonate_service_account": null, - "impersonate_service_account_delegates": null, - "kms_encryption_key": null, - "prefix": "toll-application", - "storage_custom_endpoint": null - }, - "hash": 2697172955 - }, - "modules": [ - { - "path": [ - "root" - ], - "outputs": {}, - "resources": {}, - "depends_on": [] - } - ] -} diff --git a/toll-infrastructure/main.tf b/toll-infrastructure/main.tf index a17048a9..c3f0ab2c 100644 --- a/toll-infrastructure/main.tf +++ b/toll-infrastructure/main.tf @@ -17,9 +17,23 @@ provider "google" { region = "europe-west1" } +provider "google-beta" { + project = "playground-272019" + region = "europe-west1" +} + resource "google_artifact_registry_repository" "toll-application-registry-repository" { + provider = google-beta + repository_id = "toll-application" format = "DOCKER" + cleanup_policies { + id = "keep-minimum-versions" + action = "KEEP" + most_recent_versions { + keep_count = 3 + } + } } resource "google_storage_bucket" "toll-application-bucket" { name = "playground-272019-toll-application"