ACLs: Remove server usage (#8126)

* ACLs: Remove server usage * Fix workflow * Fix tests * Update docs/security/access-control-lists.md Co-authored-by: Oz Katz <[email protected]> * Fix tests --------- Co-authored-by: Oz Katz <[email protected]>
treeverse · Sep 4, 2024 · 707deb2 · 707deb2
1 parent e3361bd
commit 707deb2
Show file tree

Hide file tree

Showing 35 changed files with 169 additions and 366 deletions.
diff --git a/.github/workflows/esti.yaml b/.github/workflows/esti.yaml
@@ -123,15 +123,6 @@ jobs:
           cache-to: |
             type=s3,region=us-east-1,bucket=lakefs-docker-cache,name=lakefs,mode=max
 
-      - name: Build and Push ACL server
-        uses: docker/build-push-action@v5
-        with:
-          push: true
-          tags: ${{ steps.login-ecr.outputs.registry }}/aclserver:${{ needs.gen-code.outputs.tag }}
-          build-args: VERSION=${{ needs.gen-code.outputs.tag }}
-          context: .
-          file: ./contrib/auth/acl/Dockerfile
-
   login-to-amazon-ecr:
     runs-on: ubuntu-latest
     needs: [ check-secrets ]
@@ -774,7 +765,7 @@ jobs:
         id: unique
         run: echo "value=$RANDOM" >> $GITHUB_OUTPUT
 
-      # Required for pulling latest fluffy
+      # Required for pulling fluffy image
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -835,7 +826,7 @@ jobs:
         id: unique
         run: echo "value=$RANDOM" >> $GITHUB_OUTPUT
 
-      # Required for pulling latest fluffy
+      # Required for pulling fluffy image
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -896,7 +887,7 @@ jobs:
         id: unique
         run: echo "value=$RANDOM" >> $GITHUB_OUTPUT
 
-      # Required for pulling latest fluffy
+      # Required for pulling fluffy image
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -941,7 +932,7 @@ jobs:
         id: unique
         run: echo "value=$RANDOM" >> $GITHUB_OUTPUT
 
-      # Required for pulling latest fluffy
+      # Required for pulling fluffy image
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -993,7 +984,7 @@ jobs:
         id: unique
         run: echo "value=$RANDOM" >> $GITHUB_OUTPUT
 
-      # Required for pulling latest fluffy
+      # Required for pulling fluffy image
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -1337,58 +1328,6 @@ jobs:
               })
             }
 
-  run-system-aws-s3-acl-server:
-    name: Run latest lakeFS app on AWS S3 + ACL Server
-    needs: [ deploy-image, login-to-amazon-ecr ]
-    runs-on: ubuntu-22.04
-    env:
-      TAG: ${{ needs.deploy-image.outputs.tag }}
-      REPO: ${{ needs.login-to-amazon-ecr.outputs.registry }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-    steps:
-      - name: Check-out code
-        uses: actions/checkout@v4
-
-      - name: Generate uniquifying value
-        id: unique
-        run: echo "value=$RANDOM" >> $GITHUB_OUTPUT
-
-      - name: Test lakeFS with S3 tests
-        uses: ./.github/actions/bootstrap-test-lakefs
-        with:
-          compose-file: esti/ops/docker-compose-acl.yaml
-          compose-flags: "--quiet-pull --exit-code-from=esti"
-        env:
-          LAKEFS_BLOCKSTORE_TYPE: s3
-          LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID: ${{ secrets.ESTI_AWS_ACCESS_KEY_ID }}
-          LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY: ${{ secrets.ESTI_AWS_SECRET_ACCESS_KEY }}
-          LAKEFS_DATABASE_TYPE: postgres
-          DOCKER_REG: ${{ needs.login-to-amazon-ecr.outputs.registry }}
-          ESTI_BLOCKSTORE_TYPE: s3
-          ESTI_STORAGE_NAMESPACE: s3://esti-system-testing/${{ github.run_number }}/${{ steps.unique.outputs.value }}
-          ESTI_AWS_ACCESS_KEY_ID: ${{ secrets.ESTI_AWS_ACCESS_KEY_ID }}
-          ESTI_AWS_SECRET_ACCESS_KEY: ${{ secrets.ESTI_AWS_SECRET_ACCESS_KEY }}
-          ESTI_VERSION: ${{ needs.deploy-image.outputs.tag }}
-
-      - name: Check files in S3 bucket
-        run: |
-          FILES_COUNT=`aws s3 ls s3://esti-system-testing/${{ github.run_number }}/${{ steps.unique.outputs.value }} --recursive | wc -l`
-          [ $FILES_COUNT -gt 5 ]
-
-      - name: lakeFS Logs on s3 failure
-        if: ${{ failure() }}
-        continue-on-error: true
-        run: docker compose -f esti/ops/docker-compose-acl.yaml logs --tail=1000 lakefs
-
-      - name: Export DB
-        if: ${{ always() }}
-        working-directory: esti/ops
-        run: |
-          if docker compose ps -q postgres; then
-            docker compose exec -T postgres pg_dumpall --username=lakefs | gzip | aws s3 cp - s3://esti-system-testing/${{ github.run_number }}/${{ steps.unique.outputs.value }}/dump.gz
-          fi
-
   run-system-aws-s3-basic-auth:
     name: Run latest lakeFS app on AWS S3 + Basic Auth
     needs: [deploy-image, login-to-amazon-ecr]
@@ -1416,7 +1355,6 @@ jobs:
           LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID: ${{ secrets.ESTI_AWS_ACCESS_KEY_ID }}
           LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY: ${{ secrets.ESTI_AWS_SECRET_ACCESS_KEY }}
           LAKEFS_DATABASE_TYPE: postgres
-          LAKEFS_AUTH_INTERNAL_BASIC: true
           DOCKER_REG: ${{ needs.login-to-amazon-ecr.outputs.registry }}
           ESTI_BLOCKSTORE_TYPE: s3
           ESTI_STORAGE_NAMESPACE: s3://esti-system-testing/${{ github.run_number }}/${{ steps.unique.outputs.value }}

diff --git a/api/swagger.yml b/api/swagger.yml
@@ -888,7 +888,7 @@ components:
             RBAC will remain enabled on GUI if "external".  That only works
             with an external auth service.
           type: string
-          enum: [simplified, external]
+          enum: [none, simplified, external]
         login_url:
           description: primary URL to use for login.
           type: string

diff --git a/clients/java-legacy/api/openapi.yaml b/clients/java-legacy/api/openapi.yaml
diff --git a/clients/java-legacy/docs/LoginConfig.md b/clients/java-legacy/docs/LoginConfig.md
diff --git a/clients/java-legacy/src/main/java/io/lakefs/clients/api/model/LoginConfig.java b/clients/java-legacy/src/main/java/io/lakefs/clients/api/model/LoginConfig.java
diff --git a/clients/java/api/openapi.yaml b/clients/java/api/openapi.yaml
diff --git a/clients/java/docs/LoginConfig.md b/clients/java/docs/LoginConfig.md
diff --git a/clients/java/src/main/java/io/lakefs/clients/sdk/model/LoginConfig.java b/clients/java/src/main/java/io/lakefs/clients/sdk/model/LoginConfig.java
diff --git a/clients/python-legacy/lakefs_client/model/login_config.py b/clients/python-legacy/lakefs_client/model/login_config.py
diff --git a/clients/python/lakefs_sdk/models/login_config.py b/clients/python/lakefs_sdk/models/login_config.py
diff --git a/clients/python/test/test_login_config.py b/clients/python/test/test_login_config.py
diff --git a/clients/python/test/test_setup_state.py b/clients/python/test/test_setup_state.py
diff --git a/clients/rust/src/models/login_config.rs b/clients/rust/src/models/login_config.rs
diff --git a/cmd/lakectl/cmd/auth.go b/cmd/lakectl/cmd/auth.go
@@ -7,7 +7,8 @@ import (
 var authCmd = &cobra.Command{
 	Use:   "auth [sub-command]",
 	Short: "Manage authentication and authorization",
-	Long:  "manage authentication and authorization including users, groups and ACLs",
+	Long: `Manage authentication and authorization including users, groups and ACLs
+This functionality is supported with an external auth service only.`,
 }
 
 func addPaginationFlags(cmd *cobra.Command) {

diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go
@@ -18,7 +18,6 @@ import (
 	"github.com/go-co-op/gocron"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/treeverse/lakefs/contrib/auth/acl"
 	"github.com/treeverse/lakefs/pkg/actions"
 	"github.com/treeverse/lakefs/pkg/api"
 	"github.com/treeverse/lakefs/pkg/auth"
@@ -104,34 +103,27 @@ Please run "lakefs superuser -h" and follow the instructions on how to migrate a
 				logger.WithError(err).Fatal("basic auth migration failed")
 			}
 		}
-
-		return auth.NewMonitoredAuthServiceAndInviter(apiService)
-	}
-	if cfg.IsAuthTypeAPI() {
-		apiService, err := auth.NewAPIAuthService(
-			cfg.Auth.API.Endpoint,
-			cfg.Auth.API.Token.SecureValue(),
-			cfg.Auth.AuthenticationAPI.ExternalPrincipalsEnabled,
-			secretStore,
-			authparams.ServiceCache(cfg.Auth.Cache),
-			logger.WithField("service", "auth_api"),
-		)
-		if err != nil {
-			logger.WithError(err).Fatal("failed to create authentication service")
-		}
-		if !cfg.Auth.API.SkipHealthCheck {
-			if err := apiService.CheckHealth(ctx, logger, cfg.Auth.API.HealthCheckTimeout); err != nil {
-				logger.WithError(err).Fatal("Auth API health check failed")
-			}
-		}
-		return auth.NewMonitoredAuthServiceAndInviter(apiService)
+		return auth.NewMonitoredAuthService(apiService)
 	}
-	authService := acl.NewAuthService(
-		kvStore,
+
+	// Not Basic - using auth server
+	apiService, err := auth.NewAPIAuthService(
+		cfg.Auth.API.Endpoint,
+		cfg.Auth.API.Token.SecureValue(),
+		cfg.Auth.AuthenticationAPI.ExternalPrincipalsEnabled,
 		secretStore,
 		authparams.ServiceCache(cfg.Auth.Cache),
+		logger.WithField("service", "auth_api"),
 	)
-	return auth.NewMonitoredAuthService(authService)
+	if err != nil {
+		logger.WithError(err).Fatal("failed to create authentication service")
+	}
+	if !cfg.Auth.API.SkipHealthCheck {
+		if err := apiService.CheckHealth(ctx, logger, cfg.Auth.API.HealthCheckTimeout); err != nil {
+			logger.WithError(err).Fatal("Auth API health check failed")
+		}
+	}
+	return auth.NewMonitoredAuthServiceAndInviter(apiService)
 }
 
 var runCmd = &cobra.Command{

diff --git a/cmd/lakefs/cmd/run_test.go b/cmd/lakefs/cmd/run_test.go
@@ -7,6 +7,7 @@ import (
 	"github.com/treeverse/lakefs/cmd/lakefs/cmd"
 	"github.com/treeverse/lakefs/pkg/auth"
 	"github.com/treeverse/lakefs/pkg/config"
+	"github.com/treeverse/lakefs/pkg/kv/kvtest"
 	"github.com/treeverse/lakefs/pkg/logging"
 )
 
@@ -23,8 +24,10 @@ func TestGetAuthService(t *testing.T) {
 	})
 	t.Run("maintain_service", func(t *testing.T) {
 		cfg := &config.Config{}
-		cfg.Auth.UIConfig.RBAC = config.AuthRBACSimplified
-		service := cmd.NewAuthService(context.Background(), cfg, logging.ContextUnavailable(), nil, nil)
+		kvStore := kvtest.GetStore(context.Background(), t)
+		meta := auth.NewKVMetadataManager("serve_test", cfg.Installation.FixedID, cfg.Database.Type, kvStore)
+		cfg.Auth.UIConfig.RBAC = config.AuthRBACNone
+		service := cmd.NewAuthService(context.Background(), cfg, logging.ContextUnavailable(), kvStore, meta)
 		_, ok := service.(auth.EmailInviter)
 		if ok {
 			t.Fatalf("expected Service to not be of type EmailInviter")