aws hubs: consistent setup of cloud permissions and bucket envs

2i2c-org · Apr 22, 2024 · 90cd85e · 90cd85e
1 parent 29febeb
commit 90cd85e
Show file tree

Hide file tree

Showing 31 changed files with 123 additions and 46 deletions.
diff --git a/config/clusters/2i2c-aws-us/dask-staging.values.yaml b/config/clusters/2i2c-aws-us/dask-staging.values.yaml
@@ -30,7 +30,9 @@ basehub:
     singleuser:
       image:
         name: pangeo/pangeo-notebook
-        tag: "2022.06.02"
+        tag: "latest"
+      extraEnv:
+        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-dask-staging/$(JUPYTERHUB_USER)
     hub:
       config:
         JupyterHub:

diff --git a/config/clusters/2i2c-aws-us/go-bgc.values.yaml b/config/clusters/2i2c-aws-us/go-bgc.values.yaml
@@ -49,8 +49,7 @@ jupyterhub:
       name: pangeo/pangeo-notebook
       tag: "2023.07.05"
     extraEnv:
-      SCRATCH_BUCKET: s3://scratch-go-bgc/$(JUPYTERHUB_USER)
-      PANGEO_SCRATCH: s3://scratch-go-bgc/$(JUPYTERHUB_USER)
+      SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-go-bgc/$(JUPYTERHUB_USER)
     profileList:
       # NOTE: About node sharing
       #

diff --git a/config/clusters/2i2c-aws-us/itcoocean.values.yaml b/config/clusters/2i2c-aws-us/itcoocean.values.yaml
@@ -79,8 +79,7 @@ jupyterhub:
             mountPath: /home/jovyan/shared-public
             subPath: _shared-public
     extraEnv:
-      SCRATCH_BUCKET: s3://scratch-itcoocean/$(JUPYTERHUB_USER)
-      PANGEO_SCRATCH: s3://scratch-itcoocean/$(JUPYTERHUB_USER)
+      SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-itcoocean/$(JUPYTERHUB_USER)
     profileList:
       # NOTE: About node sharing
       #

diff --git a/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml b/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml
@@ -52,6 +52,8 @@ basehub:
         # pangeo/pangeo-notebook is maintained at: https://github.com/pangeo-data/pangeo-docker-images
         name: pangeo/pangeo-notebook
         tag: "2023.05.18"
+      extraEnv:
+        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-ncar-cisl/$(JUPYTERHUB_USER)
       profileList:
         # NOTE: About node sharing
         #

diff --git a/config/clusters/2i2c-aws-us/showcase.values.yaml b/config/clusters/2i2c-aws-us/showcase.values.yaml
@@ -50,8 +50,7 @@ basehub:
           enable_auth_state: true
     singleuser:
       extraEnv:
-        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-researchdelight/$(JUPYTERHUB_USER)
-        PANGEO_SCRATCH: s3://2i2c-aws-us-scratch-researchdelight/$(JUPYTERHUB_USER)
+        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-showcase/$(JUPYTERHUB_USER)
         PERSISTENT_BUCKET: s3://2i2c-aws-us-persistent-showcase/$(JUPYTERHUB_USER)
         GH_SCOPED_CREDS_CLIENT_ID: Iv1.f9261c4c78b4dfdd
         GH_SCOPED_CREDS_APP_URL: https://github.com/apps/2i2c-community-showcase-hub

diff --git a/config/clusters/2i2c-aws-us/staging.values.yaml b/config/clusters/2i2c-aws-us/staging.values.yaml
@@ -32,3 +32,6 @@ jupyterhub:
         authenticator_class: "github"
       GitHubOAuthenticator:
         oauth_callback_url: "https://staging.aws.2i2c.cloud/hub/oauth_callback"
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/earthscope/prod.values.yaml b/config/clusters/earthscope/prod.values.yaml
@@ -1,4 +1,8 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::762698921361:role/earthscope-prod
+
   jupyterhub:
     ingress:
       hosts: [geolab.earthscope.cloud]
@@ -19,3 +23,6 @@ basehub:
           extra_authorize_params:
             # This isn't an actual URL, just a string. Must not have a trailing slash
             audience: https://api.earthscope.org
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://earthscope-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/earthscope/staging.values.yaml b/config/clusters/earthscope/staging.values.yaml
@@ -1,4 +1,8 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::762698921361:role/earthscope-staging
+
   jupyterhub:
     ingress:
       hosts:
@@ -20,3 +24,6 @@ basehub:
           extra_authorize_params:
             # This isn't an actual URL, just a string. Must not have a trailing slash
             audience: https://api.dev.earthscope.org
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://earthscope-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/gridsst/prod.values.yaml b/config/clusters/gridsst/prod.values.yaml
@@ -12,3 +12,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://gridsst.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://gridsst-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/gridsst/staging.values.yaml b/config/clusters/gridsst/staging.values.yaml
@@ -12,3 +12,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://staging.gridsst.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://gridsst-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/jupyter-health/prod.values.yaml b/config/clusters/jupyter-health/prod.values.yaml
@@ -11,3 +11,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://jupyter-health.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://jupyter-health-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/jupyter-health/staging.values.yaml b/config/clusters/jupyter-health/staging.values.yaml
@@ -11,3 +11,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://staging.jupyter-health.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://jupyter-health-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/jupyter-meets-the-earth/staging.values.yaml b/config/clusters/jupyter-meets-the-earth/staging.values.yaml
@@ -3,18 +3,15 @@ basehub:
     annotations:
       eks.amazonaws.com/role-arn: arn:aws:iam::286354552638:role/jupyter-meets-the-earth-staging
   jupyterhub:
-    hub:
-      config:
-        CILogonOAuthenticator:
-          oauth_callback_url: "https://staging.jmte.2i2c.cloud/hub/oauth_callback"
     ingress:
       hosts: [staging.jmte.2i2c.cloud]
       tls:
         - hosts: [staging.jmte.2i2c.cloud]
           secretName: https-auto-tls
-
+    hub:
+      config:
+        CILogonOAuthenticator:
+          oauth_callback_url: "https://staging.jmte.2i2c.cloud/hub/oauth_callback"
     singleuser:
       extraEnv:
-        # This bucket is created via terraform.
-        SCRATCH_BUCKET: s3://jupyter-meets-the-earth-staging-scratch/$(JUPYTERHUB_USER)
-        PANGEO_SCRATCH: s3://jupyter-meets-the-earth-staging-scratch/$(JUPYTERHUB_USER)
+        SCRATCH_BUCKET: s3://jupyter-meets-the-earth-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/kitware/prod.values.yaml b/config/clusters/kitware/prod.values.yaml
@@ -12,3 +12,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://kitware.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://kitware-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/kitware/staging.values.yaml b/config/clusters/kitware/staging.values.yaml
@@ -12,3 +12,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://staging.kitware.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://kitware-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-esdis/prod.values.yaml b/config/clusters/nasa-esdis/prod.values.yaml
@@ -7,10 +7,10 @@ jupyterhub:
     tls:
       - hosts: [esdis.2i2c.cloud]
         secretName: https-auto-tls
-  singleuser:
-    extraEnv:
-      SCRATCH_BUCKET: s3://nasa-esdis-scratch/$(JUPYTERHUB_USER)
   hub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: "https://esdis.2i2c.cloud/hub/oauth_callback"
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://nasa-esdis-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-esdis/staging.values.yaml b/config/clusters/nasa-esdis/staging.values.yaml
@@ -7,10 +7,10 @@ jupyterhub:
     tls:
       - hosts: [staging.esdis.2i2c.cloud]
         secretName: https-auto-tls
-  singleuser:
-    extraEnv:
-      SCRATCH_BUCKET: s3://nasa-esdis-scratch-staging/$(JUPYTERHUB_USER)
   hub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: "https://staging.esdis.2i2c.cloud/hub/oauth_callback"
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://nasa-esdis-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-ghg/prod.values.yaml b/config/clusters/nasa-ghg/prod.values.yaml
@@ -16,3 +16,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://hub.ghg.center/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-ghg-hub-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-ghg/staging.values.yaml b/config/clusters/nasa-ghg/staging.values.yaml
@@ -16,3 +16,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://staging.ghg.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-ghg-hub-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml
@@ -12,3 +12,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://hub.openveda.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-veda-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-veda/staging.values.yaml b/config/clusters/nasa-veda/staging.values.yaml
@@ -12,3 +12,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://staging.hub.openveda.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-veda-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/opensci/staging.values.yaml b/config/clusters/opensci/staging.values.yaml
@@ -1,3 +1,8 @@
+userServiceAccount:
+  enabled: true
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::211125293633:role/opensci-staging
+
 jupyterhub:
   ingress:
     hosts:
@@ -28,6 +33,8 @@ jupyterhub:
           name: ""
           url: ""
   singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://opensci-scratch-staging/$(JUPYTERHUB_USER)
     profileList:
       - display_name: "Only Profile Available, this info is not shown in the UI"
         slug: only-choice

diff --git a/config/clusters/smithsonian/prod.values.yaml b/config/clusters/smithsonian/prod.values.yaml
@@ -1,7 +1,19 @@
 basehub:
+  userServiceAccount:
+    enabled: true
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::969396938818:role/smithsonian-prod
+
   jupyterhub:
     ingress:
       hosts: [smithsonian.2i2c.cloud]
       tls:
         - hosts: [smithsonian.2i2c.cloud]
           secretName: https-auto-tls
+    hub:
+      config:
+        GitHubOAuthenticator:
+          oauth_callback_url: https://smithsonian.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://smithsonian-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/smithsonian/staging.values.yaml b/config/clusters/smithsonian/staging.values.yaml
@@ -1,7 +1,19 @@
 basehub:
+  userServiceAccount:
+    enabled: true
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::969396938818:role/smithsonian-staging
+
   jupyterhub:
     ingress:
       hosts: [staging.smithsonian.2i2c.cloud]
       tls:
         - hosts: [staging.smithsonian.2i2c.cloud]
           secretName: https-auto-tls
+    hub:
+      config:
+        GitHubOAuthenticator:
+          oauth_callback_url: https://staging.smithsonian.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://smithsonian-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/ubc-eoas/prod.values.yaml b/config/clusters/ubc-eoas/prod.values.yaml
@@ -1,3 +1,8 @@
+userServiceAccount:
+  enabled: true
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::259060176665:role/ubc-eoas-prod
+
 jupyterhub:
   ingress:
     hosts: [ubc-eoas.2i2c.cloud]
@@ -8,3 +13,6 @@ jupyterhub:
     config:
       CILogonOAuthenticator:
         oauth_callback_url: https://ubc-eoas.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://ubc-eoas-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/ubc-eoas/staging.values.yaml b/config/clusters/ubc-eoas/staging.values.yaml
@@ -1,3 +1,8 @@
+userServiceAccount:
+  enabled: true
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::259060176665:role/ubc-eoas-staging
+
 jupyterhub:
   ingress:
     hosts: [staging.ubc-eoas.2i2c.cloud]
@@ -8,3 +13,6 @@ jupyterhub:
     config:
       CILogonOAuthenticator:
         oauth_callback_url: https://staging.ubc-eoas.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://ubc-eoas-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/victor/prod.values.yaml b/config/clusters/victor/prod.values.yaml
@@ -13,3 +13,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://victor.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://victor-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/victor/staging.values.yaml b/config/clusters/victor/staging.values.yaml
@@ -14,6 +14,8 @@ basehub:
         GitHubOAuthenticator:
           oauth_callback_url: https://staging.victor.2i2c.cloud/hub/oauth_callback
     singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://victor-scratch-staging/$(JUPYTERHUB_USER)
       profileList:
         # Create a small instance that can launch a custom image
         - display_name: "Bring your own image - Small: m5.large"

diff --git a/terraform/aws/projects/2i2c-aws-us.tfvars b/terraform/aws/projects/2i2c-aws-us.tfvars
@@ -11,7 +11,7 @@ user_buckets = {
   "scratch-dask-staging" : {
     "delete_after" : 7
   },
-  "scratch-researchdelight" : {
+  "scratch-showcase" : {
     "delete_after" : 7
   },
   "persistent-showcase" : {
@@ -43,7 +43,7 @@ hub_cloud_permissions = {
   "showcase" : {
     "user-sa" : {
       bucket_admin_access : [
-        "scratch-researchdelight",
+        "scratch-showcase",
         "persistent-showcase",
       ],
     },

diff --git a/terraform/aws/projects/catalystproject-africa.tfvars b/terraform/aws/projects/catalystproject-africa.tfvars
@@ -3,26 +3,3 @@ region = "af-south-1"
 cluster_name = "catalystproject-africa"
 
 cluster_nodes_location = "af-south-1a"
-
-user_buckets = {
-  "scratch-staging" : {
-    "delete_after" : 7
-  },
-  "scratch" : {
-    "delete_after" : 7
-  },
-}
-
-
-hub_cloud_permissions = {
-  "staging" : {
-    "user-sa" : {
-      bucket_admin_access : ["scratch-staging"],
-    },
-  },
-  "prod" : {
-    "user-sa" : {
-      bucket_admin_access : ["scratch"],
-    },
-  },
-}
diff --git a/terraform/aws/projects/jupyter-meets-the-earth.tfvars b/terraform/aws/projects/jupyter-meets-the-earth.tfvars
@@ -8,6 +8,9 @@ user_buckets = {
   "scratch-staging" : {
     "delete_after" : 7
   },
+  // IMPORTANT: This bucket isn't used, they are instead using s3://jmte-scratch
+  //            that doesn't have a delete_after policy setup etc, but maybe
+  //            they want to have.
   "scratch" : {
     "delete_after" : 7
   },