Merge 2023 updates to stable (#89)

* Test of directly syncing specific tide model subfolders (#86) * Test of directly syncing specific tide model subfolders * Automatically update integration test validation results --------- Co-authored-by: robbibt <[email protected]> * Automatically update integration test validation results * 2023 update (#87) * Update notebooks * 2023 update * Fix pep * Clean notebook * Update docker image base * Update vector.py * Automatically update integration test validation results * Add experiment database access * Update * Update after permissions * Update dockerfile and CI * Update docker * Remove AWS CLI * Add test env * Simplify by installing pytest in setup.py * Add file copying to workflow * Try new paths * Check everything works up to copy step * Try ls * Fix * Try combined approach * Try smaller tests * Fix paths * Automatically update integration test validation results * Try changing locations files are written to * Fix * Update .gitignore * Automatically update integration test validation results * Update stats_tests.csv * Automatically update integration test validation results * Clean repo * Update gitignore * Update test_coastline.py * Update test_coastline.py * Update test_coastline.py * Debug * Debug * Finally fix validation bug * Automatically update integration test validation results * Fix formatting * Fix pep8 * Add image build and artifacts * Automatically update integration test validation results * Update vector.py * Automatically update integration test validation results --------- Co-authored-by: robbibt <[email protected]> * Fix branch reference * Update badge * Update Dockerfile * Automatically update integration test validation results * Update dea-coastlines-image.yaml * Fix styling bug in 2023 uncertain shorelines (#93) * Add fixed style * Automatically update integration test validation results * Update dea-coastlines-image.yaml * Automatically update integration test validation results --------- Co-authored-by: robbibt <[email protected]> * Try old approach for image tagging * Fix workflow name * Update dea-coastlines-image.yaml * Use simpler tagging method that doesn't require git history * Add notebook for automatically generating `.sld` XML styles for GeoServer (#94) * Add notebook for autogenerating coastlines styles * Add notebook for generating XML styles for Geoserver * Revert CLI notebook --------- Co-authored-by: robbibt <[email protected]> Co-authored-by: ClaireP <[email protected]>
GeoscienceAustralia · Aug 22, 2024 · b7d07c0 · b7d07c0
1 parent 9992762
commit b7d07c0
Show file tree

Hide file tree

Showing 33 changed files with 6,277 additions and 8,902 deletions.
diff --git a/.github/workflows/dea-coastlines-image.yaml b/.github/workflows/dea-coastlines-image.yaml
@@ -0,0 +1,176 @@
+name: Image build and test
+
+env:
+  IMAGE_NAME: geoscienceaustralia/dea-coastlines
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'coastlines/**'
+      - 'data/**'
+      - 'tests/**'
+      - '.github/workflows/dea-coastlines-image.yaml'
+      - 'Dockerfile'
+      - 'requirements.in'
+      - 'requirements.txt'
+      - 'setup.py'
+      - 'codecov.yaml'
+
+  pull_request:
+    branches:
+      - develop
+    paths:
+      - 'coastlines/**'
+      - 'data/**'
+      - 'tests/**'
+      - '.github/workflows/dea-coastlines-image.yaml'
+      - 'Dockerfile'
+      - 'requirements.in'
+      - 'requirements.txt'
+      - 'setup.py'
+      - 'codecov.yaml'
+
+  release:
+    types: [edited, published]
+
+permissions:
+  id-token: write  # Required for requesting Json web token
+  contents: write  # Required for actions/checkout
+  pull-requests: write  # Required for validation results comment bot
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox
+          aws-region: ap-southeast-2
+          role-duration-seconds: 7200  # 2 hours
+
+      - name: Get database credentials
+        run: |
+          username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption  --query Parameter.Value --output text)
+          echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV
+
+      - name: Open port forward to RDS
+        run: |
+          npx basti connect \
+            --custom-target-vpc vpc-086904199e505c1f6 \
+            --custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \
+            --custom-target-port 5432 \
+            --local-port 5432 &
+          # Wait until the connection comes up, but, if it doesn't, don't hang forever.
+          npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432
+          echo "PGPORT=5432" >> $GITHUB_ENV
+          echo "PGHOST=localhost" >> $GITHUB_ENV
+
+      - name: Build DEA Coastlines docker image 
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          docker build -t dea_coastlines .
+
+      - name: Run integration tests
+        run: |
+          # Download tide modelling files and unzip
+          wget --no-verbose https://www.dropbox.com/s/uemd8ib2vfw5nad/tide_models.zip?dl=1 -O tide_models.zip
+          unzip -q tide_models.zip
+
+          # Run integration tests using Docker
+          docker run \
+            --net=host \
+            --env DATACUBE_DB_URL \
+            --env AWS_SESSION_TOKEN \
+            --env AWS_REGION \
+            --env AWS_ACCESS_KEY_ID \
+            --env AWS_SECRET_ACCESS_KEY \
+            --env AWS_SESSION_TOKEN \
+            --volume ${GITHUB_WORKSPACE}:/code \
+            --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \
+            dea_coastlines pytest -v --cov=coastlines --cov-report=xml tests
+
+      # Commit validation results produced by integration tests back into repo
+      - name: Commit validation results into repository
+        uses: stefanzweifel/git-auto-commit-action@v4
+        if: github.event_name == 'pull_request'
+        continue-on-error: true
+        with:
+          commit_message: Automatically update integration test validation results
+          file_pattern: 'tests/stats_tests.png tests/stats_tests.csv tests/README.md'
+
+      # Post validation tesults as comment on PR
+      - name: Post validation results as comment
+        uses: mshick/add-pr-comment@v2
+        if: github.event_name == 'pull_request'
+        with:
+          message: |
+            ![](https://github.com/GeoscienceAustralia/dea-coastlines/blob/${{ github.head_ref }}/tests/stats_tests.png?raw=true")
+            
+            For full integration test results, refer to the [Tests directory README](https://github.com/GeoscienceAustralia/dea-coastlines/tree/${{ github.head_ref }}/tests).
+
+      # Upload code coverage results
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      # Add output GeoPackage data to Actions artifact so it can be downloaded
+      - uses: actions/upload-artifact@v3
+        with:
+          name: output_geopackage
+          path: ./data/processed/tests/coastlines_tests.gpkg
+
+      # Add output ESRI Shapefile data to Actions artifact so it can be downloaded
+      - uses: actions/upload-artifact@v3
+        with:
+          name: output_shps
+          path: ./data/processed/tests/coastlines_tests.shp.zip
+
+  push_ecr:
+    needs: [test]
+    runs-on: ubuntu-latest
+
+    # Only run on a push to the develop branch OR a release
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/develop') || (github.event_name == 'release')
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # If action is triggered by a release, tag image
+      # using a custom image tag extracted from the release
+      - name: Get current version tag from release to use as image tag
+        if: github.event_name == 'release'
+        run: |
+          echo "RELEASE=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV
+
+      # If action is trigged by a push (not release), tag image
+      # using a tag based on the most recent commit hash
+      - name: Get git commit hash for push to branch to use as image tag
+        if: github.event_name != 'release'
+        run: |
+          echo "TAG=dev$(git rev-parse --short HEAD)" \
+          >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::538673716275:role/github-actions-role
+          aws-region: ap-southeast-2
+
+      - name: Push image to ECR
+        uses: whoan/docker-build-with-cache-action@master
+        with:
+          context: ./
+          registry: 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com
+          image_name: ${{ env.IMAGE_NAME }}
+          image_tag: latest,${{ env.RELEASE }}
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
diff --git a/.gitignore b/.gitignore
@@ -11,4 +11,6 @@
 !*.sql
 !**.github
 !*.md
+!requirements.txt
 !tests/*.png
+!tests/*.csv