Merge pull request #190 from ShivTestOrg/scraperui

fix: adds scraper logic and token handling

.dev.vars.example

@@ -0,0 +1,3 @@
+SUPABASE_URL=""
+SUPABASE_KEY=""
+VOYAGEAI_API_KEY=""

.github/sync-env-vars.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Check if required environment variables are set
+if [ -z "$CLOUDFLARE_ACCOUNT_ID" ] || [ -z "$CLOUDFLARE_API_TOKEN" ] || [ -z "$GITHUB_REPOSITORY" ] || \
+   [ -z "$VOYAGEAI_API_KEY" ] || [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_KEY" ]; then
+    echo "Error: Required environment variables are not set"
+    exit 1
+fi
+
+# Extract just the repository name from org/repo format
+REPOSITORY_NAME=$(echo "$GITHUB_REPOSITORY" | cut -d'/' -f2)
+# Replace dots with hyphens
+REPOSITORY_NAME=${REPOSITORY_NAME//./-}
+
+# Echo the repository name
+echo "Repository name: $REPOSITORY_NAME"
+
+# Make the API call to Cloudflare
+curl -X PATCH \
+  "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/pages/projects/${REPOSITORY_NAME}/deployment_configs" \
+  -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
+  -H "Content-Type: application/json" \
+  --data '{
+    "deployment_configs": {
+      "production": {
+        "env_vars": {
+          "VOYAGEAI_API_KEY": {
+            "value": "'"${VOYAGEAI_API_KEY}"'",
+            "type": "secret_text"
+          },
+          "SUPABASE_URL": {
+            "value": "'"${SUPABASE_URL}"'",
+            "type": "secret_text"
+          },
+          "SUPABASE_KEY": {
+            "value": "'"${SUPABASE_KEY}"'",
+            "type": "secret_text"
+          }
+        }
+      }
+    }
+  }'

.github/workflows/build.yml

@@ -35,4 +35,8 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: static
-          path: static
+          path: |
+              static
+              functions
+              package.json
+              yarn.lock

.github/workflows/deploy.yml

@@ -25,5 +25,25 @@ jobs:
           cloudflare_api_token: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           commit_sha: ${{ github.event.workflow_run.head_sha }}
           workflow_run_id: ${{ github.event.workflow_run.id }}
-          app_id: ${{ secrets.APP_ID }}
-          app_private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          statics_directory: "static"
+
+  update_env_secrets_in_cloudflare:
+    name: Update environment secrets in Cloudflare
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Make script executable
+        run: chmod +x .github/sync-env-vars.sh
+
+      - name: Run sync environment variables script
+        env:
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }}
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+        run: bash .github/sync-env-vars.sh
+

.gitignore

@@ -18,3 +18,6 @@ cypress/videos
 
 # vscode
 .vscode
+
+# Cloudflare
+.dev.vars

README.md

@@ -14,6 +14,12 @@ yarn start
 open http://localhost:8080
 ```
 
+## Setup Scraper Function
+
+For local development of the scraper function, you need to set up a `.dev.vars` file based on the `.dev.vars.example` file.
+
+For deployment, you would need to add the secrets to the Cloudflare Dashboard, for the worker.
+
 ### Automatic Light Mode
 
 - There is a plugin (`build/plugins/invert-colors.ts`) that inverts the greyscale shades in `style.css` and outputs `inverted-style.css`.