flex templates - 1

.github/actions/build-flex-template/action.yml

@@ -0,0 +1,35 @@
+name: Build flex template
+description: Build flex template using gcloud
+
+inputs:
+  template_file:
+    description: Google Cloud Storage location of the flex template file
+    required: true
+  image_gcr:
+    description: Google Artifact Registry location to store the flex template image to be built
+    required: true
+  artifact_name:
+    description: Artifact name
+    required: true
+  main_class:
+    description: Main class
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.artifact_name }}
+
+    - uses: "google-github-actions/setup-gcloud@v1"
+
+    - name: Flex template build
+      shell: bash
+      run: |
+        gcloud dataflow flex-template build ${{ inputs.template_file }} \
+            --image-gcr-path "${{ image_gcr }}" \
+            --sdk-language "JAVA" \
+            --flex-template-base-image JAVA17 \
+            --jar "${{ inputs.artifact_name }}" \
+            --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="${{ inputs.main_class }}" \

.github/actions/setup/action.yml

@@ -1,12 +1,11 @@
-name: 'Setup'
-description: 'Setup build'
+name: Setup
+description: Setup java
 
 runs:
-  using: "composite"
+  using: composite
   steps:
-    - name: Setup JDK
-      uses: actions/setup-java@v3
+    - uses: actions/setup-java@v3
       with:
         distribution: temurin
         java-version: 17
-        cache: sbt
+        cache: sbt

.github/workflows/automerge.yml → .github/workflows/auto-merge.yml

@@ -1,4 +1,4 @@
-name: Scala Steward auto-merge
+name: Auto-merge
 
 on: pull_request
 
@@ -10,10 +10,10 @@ jobs:
   automerge:
     runs-on: ubuntu-latest
 
-    if: ${{ github.actor == 'mkuthan-scala-steward[bot]' }}
+    if: ${{ github.actor == 'mkuthan-scala-steward[bot]' || github.actor == 'dependabot[bot]' }}
 
     steps:
-      - name: Enable auto-merge for Scala Steward PRs
+      - name: Enable auto-merge
         run: gh pr merge --auto --squash "$PR_URL"
         env:
           PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/ci.yml

@@ -1,6 +1,8 @@
 name: CI
 
-on: [push, workflow_dispatch]
+on:
+  workflow_dispatch: ~
+  pull_request: ~
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -32,25 +34,9 @@ jobs:
       - uses: ./.github/actions/setup
       - run: sbt "scalafixAll --check"
 
-  tf-validate:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dflook/terraform-validate@v1
-        with:
-          path: toll-infrastructure
-
-  tf-fmt:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dflook/terraform-fmt-check@v1
-        with:
-          path: toll-infrastructure
-
   test:
     runs-on: ubuntu-latest
-    needs: [scalafmt, scapegoat, scalafix, tf-validate, tf-fmt]
+    needs: [scalafmt, scapegoat, scalafix]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup

.github/workflows/flex-templates.yml

@@ -0,0 +1,47 @@
+name: Build flex templates
+
+on:
+  workflow_dispatch: ~
+  push:
+    branches:
+      - main
+
+jobs:
+  assembly:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+      - run: sbt tollApplication/assembly
+      - uses: actions/upload-artifact@v3
+        with:
+          name: toll-application
+          path: toll-application/target/scala-2.13/toll-application.jar"
+
+  build-batch:
+    runs-on: ubuntu-latest
+    needs: [assembly]
+    steps:
+      - uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}"
+      - uses: ./.github/actions/build-flex-template
+        with:
+          template_file: gs://playground-272019-toll-application/templates/toll-application-batch.json
+          image_gcr: europe-west1-docker.pkg.dev/playground-272019/toll-application/toll-application-batch:latest
+          artifact_name: toll-application
+          main_class: org.mkuthan.streamprocessing.toll.application.batch.TollBatchJob
+
+  build-streaming:
+    runs-on: ubuntu-latest
+    needs: [assembly]
+    steps:
+      - uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}"
+      - uses: ./.github/actions/build-flex-template
+        with:
+          template_file: gs://playground-272019-toll-application/templates/toll-application-streaming.json
+          image_gcr: europe-west1-docker.pkg.dev/playground-272019/toll-application/toll-application-streaming:latest
+          artifact_name: toll-application
+          main_class: org.mkuthan.streamprocessing.toll.application.streaming.TollStreamingJob

.github/workflows/scala-steward.yml

@@ -1,9 +1,9 @@
 name: Scala Steward
 
 on:
+  workflow_dispatch: ~
   schedule:
     - cron: "0 0 * * 0" # 00:00 every Sunday
-  workflow_dispatch:
 
 jobs:
   scala-steward:

.github/workflows/tf-apply.yml

@@ -0,0 +1,26 @@
+name: Terraform apply
+
+on:
+  workflow_dispatch: ~
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  pull-requests: write
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  apply:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}"
+      - uses: dflook/terraform-apply@v1
+        with:
+          path: toll-infrastructure

.github/workflows/tf-plan.yml

@@ -1,25 +1,41 @@
 name: Terraform plan
 
-on: [pull_request]
+on:
+  workflow_dispatch: ~
+  pull_request: ~
 
 permissions:
   contents: read
   pull-requests: write
 
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
 jobs:
-  plan:
+  validate:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dflook/terraform-validate@v1
+        with:
+          path: toll-infrastructure
 
-    env:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
+  fmt:
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: dflook/terraform-fmt-check@v1
+        with:
+          path: toll-infrastructure
 
+  plan:
+    runs-on: ubuntu-latest
+    needs: [validate, fmt]
+    steps:
+      - uses: actions/checkout@v4
       - uses: google-github-actions/auth@v1
         with:
           credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}"
-
       - uses: dflook/terraform-plan@v1
         with:
           path: toll-infrastructure

README.md

@@ -11,9 +11,8 @@ Shared modules:
 
 Use cases:
 
-* `toll-application`, `toll-domain` - sample application for toll data processing,
+* `toll-application`, `toll-domain`, `toll-infrastructure` - sample application for toll data processing,
 see [blog post](https://mkuthan.github.io/blog/2023/09/27/unified-batch-streaming/)
-* `toll-infrastructure` - GCP infrastructure as a code for sample application
 * `word-count` - fixed window example,
 see [blog post](https://mkuthan.github.io/blog/2022/01/28/stream-processing-part1/)
 * `session-window` - session window example,

toll-application/run-batch.sh

@@ -1,15 +1,17 @@
 #!/bin/bash
 
+DATE=`date +%Y-%m-%d`
+
 PROJECT=playground-272019
 REGION=europe-west1
-DATE=`date +%Y-%m-%d`
 
 gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \
     --template-file-gcs-location "gs://$PROJECT-toll-application/templates/toll-application-batch.json" \
     --region "$REGION" \
-    --staging-location="gs://$PROJECT-toll-application/staging/" \
-    --additional-experiments="use_runner_v2" \
-    --max-workers=1 \
+    --staging-location "gs://$PROJECT-toll-application/staging/" \
+    --additional-experiments "use_runner_v2" \
+    --max-workers 1 \
+    --worker-machine-type "t2d-standard-1" \
     --parameters effectiveDate="$DATE" \
     --parameters entryTable="$PROJECT.toll_application.toll-booth-entry" \
     --parameters exitTable="$PROJECT.toll_application.toll-booth-exit" \
@@ -18,5 +20,5 @@ gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \
     --parameters entryStatsDailyTable="$PROJECT.toll_application.toll-booth-entry-stats-daily" \
     --parameters totalVehicleTimesOneHourGapTable="$PROJECT.toll_application.total-vehicle-times-one-hour-gap" \
     --parameters totalVehicleTimesOneHourGapDiagnosticTable="$PROJECT.toll_application.total-vehicle-times-one-hour-gap-diagnostic" \
-    --parameters vehiclesWithExpiredRegistrationDailyTable="$PROJECT.toll_application.vehicles-with-expired-registration-diagnostic-daily" \
+    --parameters vehiclesWithExpiredRegistrationDailyTable="$PROJECT.toll_application.vehicles-with-expired-registration-daily" \
     --parameters vehiclesWithExpiredRegistrationDailyDiagnosticTable="$PROJECT.toll_application.vehicles-with-expired-registration-daily-diagnostic" \

toll-application/run-streaming.sh

@@ -6,10 +6,11 @@ REGION=europe-west1
 gcloud dataflow flex-template run "toll-application-`date +%Y%m%d-%H%M%S`" \
     --template-file-gcs-location "gs://$PROJECT-toll-application/templates/toll-application-streaming.json" \
     --region "$REGION" \
-    --staging-location="gs://$PROJECT-toll-application/staging/" \
+    --staging-location "gs://$PROJECT-toll-application/staging/" \
     --enable-streaming-engine \
-    --additional-experiments="use_runner_v2" \
-    --max-workers=1 \
+    --additional-experiments "use_runner_v2" \
+    --max-workers 1 \
+    --worker-machine-type "t2d-standard-1" \
     --parameters entrySubscription="projects/$PROJECT/subscriptions/toll-booth-entry" \
     --parameters entryDlq="gs://$PROJECT-toll-application/dlq/entry" \
     --parameters exitSubscription="projects/$PROJECT/subscriptions/toll-booth-exit" \