Merge pull request #103 from technologiestiftung/staging

Staging to Master prepare release v2.0.0

.github/workflows/test-harvest.yml

@@ -60,7 +60,7 @@ jobs:
         id: api-start
         run: cd api && supabase start | grep -w "service_role key" | cut -d ":" -f 2 | xargs |  tr -d '\n' | awk '{print "service_role_key="$1}' >> "$GITHUB_OUTPUT" && cd ..
       - name: run the harvester
-        run: docker run --env PG_SERVER='0.0.0.0' --env PG_DB --env PG_PORT --env PG_USER --env PG_PASS --env SUPABASE_URL --env SUPABASE_ACCESS_TOKEN='${{ steps.api-start.outputs.service_role_key }}' --env SUPABASE_BUCKET_NAME --env MAPBOXTOKEN --env MAPBOXUSERNAME --env LOGGING --env OUTPUT --network host technologiestiftung/giessdenkiez-de-dwd-harvester:test
+        run: docker run --env PG_SERVER='0.0.0.0' --env PG_DB --env PG_PORT --env PG_USER --env PG_PASS --env SUPABASE_URL --env SUPABASE_SERVICE_ROLE_KEY='${{ steps.api-start.outputs.service_role_key }}' --env SUPABASE_BUCKET_NAME --env MAPBOXTOKEN --env MAPBOXUSERNAME --env LOGGING --env OUTPUT --network host technologiestiftung/giessdenkiez-de-dwd-harvester:test
       - name: stop the api
         run: cd api && supabase stop && cd ..
   release:

action.yml

@@ -20,9 +20,9 @@ inputs:
   SUPABASE_URL:
     required: true
     description: "The base URL to Supabase, e.g. http://localhost:54321 for a local Supabase"
-  SUPABASE_ACCESS_TOKEN:
+  SUPABASE_SERVICE_ROLE_KEY:
     required: true
-    description: "The access token for accessing the Supabase bucket"
+    description: "The service role key for accessing the Supabase bucket"
   SUPABASE_BUCKET_NAME:
     description: "The bucket name where the harvested data should be pushed to"
     required: true
@@ -56,7 +56,7 @@ runs:
     PG_PASS: ${{ inputs.PG_PASS }}
     PG_DB: ${{ inputs.PG_DB }}
     SUPABASE_URL: ${{ inputs.SUPABASE_URL }}
-    SUPABASE_ACCESS_TOKEN: ${{ inputs.SUPABASE_ACCESS_TOKEN }}
+    SUPABASE_SERVICE_ROLE_KEY: ${{ inputs.SUPABASE_SERVICE_ROLE_KEY }}
     SUPABASE_BUCKET_NAME: ${{ inputs.SUPABASE_BUCKET_NAME }}
     MAPBOXUSERNAME: ${{ inputs.MAPBOXUSERNAME }}
     MAPBOXTOKEN: ${{ inputs.MAPBOXTOKEN }}

docker-compose.yml

@@ -17,7 +17,7 @@ services:
       # PG_USER: fangorn
       # PG_PASS: ent
       # SUPABASE_PROJECT_ID=
-      # SUPABASE_ACCESS_TOKEN=
+      # SUPABASE_SERVICE_ROLE_KEY=
       # SUPABASE_BUCKET_NAME=data_assets
       # bygiessdenkiez-devoverwriting these here we make sure
       # we dont push all the time to mapbox 💵
@@ -31,7 +31,7 @@ services:
     # By overtwriting the command we can start the container
     # move into it (docker exec -it {CONTAINER ID} /bin/sh
     # and execute or debug all the scripts manually
-    command: ["tail", "-f", "/dev/null"]
+    command: [ "tail", "-f", "/dev/null" ]
 
   postgres:
     container_name: "postgres"

harvester/harvester.py

@@ -39,7 +39,7 @@
 load_dotenv()
 
 # check if all required environmental variables are accessible
-for env_var in ["PG_DB", "PG_PORT", "PG_USER", "PG_PASS", "SUPABASE_URL", "SUPABASE_BUCKET_NAME", "SUPABASE_ACCESS_TOKEN"]:
+for env_var in ["PG_DB", "PG_PORT", "PG_USER", "PG_PASS", "SUPABASE_URL", "SUPABASE_BUCKET_NAME", "SUPABASE_SERVICE_ROLE_KEY"]:
     if env_var not in os.environ:
         logging.error(
             "❌Environmental Variable {} does not exist".format(env_var))
@@ -60,7 +60,8 @@
 
 SUPABASE_URL = os.getenv('SUPABASE_URL')
 SUPABASE_BUCKET_NAME = os.getenv('SUPABASE_BUCKET_NAME')
-SUPABASE_ACCESS_TOKEN = os.getenv('SUPABASE_ACCESS_TOKEN')
+SUPABASE_SERVICE_ROLE_KEY = os.getenv('SUPABASE_SERVICE_ROLE_KEY')
+
 
 try:
     conn = psycopg2.connect(dsn)
@@ -316,9 +317,9 @@ def upload_file_to_supabase_storage(file_path, file_name):
             r = requests.put if check_file_exists_in_supabase_storage(file_name) else requests.post
             response = r(
                 file_url,
-                files={'file': file}, 
+                files={'file': file},
                 headers={
-                    'Authorization': f'Bearer {SUPABASE_ACCESS_TOKEN}',
+                    'Authorization': f'Bearer {SUPABASE_SERVICE_ROLE_KEY}',
                     'ContentType': 'application/geo+json',
                     'AcceptEncoding': 'gzip, deflate, br'
                 },
@@ -330,7 +331,7 @@ def upload_file_to_supabase_storage(file_path, file_name):
                 logging.warning(response.status_code)
                 logging.warning(response.content)
                 logging.warning("❌ Could not upload {} to supabase storage".format(file_name))
-
+                
         except Exception as error:
             logging.warning(error)
             logging.warning("❌ Could not upload {} supabase storage".format(file_name))

harvester/sample.env

@@ -7,7 +7,8 @@ PG_DB=
 
 # Supabase Storage access data to store the resulting geojson/csv files
 SUPABASE_URL=
-SUPABASE_ACCESS_TOKEN=
+SUPABASE_SERVICE_ROLE_KEY=
+
 SUPABASE_BUCKET_NAME=
 
 # Generation of maptiles through mapbox requires api credentials and desired tileset name
@@ -26,6 +27,7 @@ LOGGING=INFO
 # PG_PASS=postgres_password
 # PG_DB=postgres
 # SUPABASE_URL=
-# SUPABASE_ACCESS_TOKEN=
+# SUPABASE_SERVICE_ROLE_KEY=
+
 # SUPABASE_BUCKET_NAME=
 # OUTPUT=True