refactor: testing idea to wrap coscheduling

.github/test-kind-config.yaml

@@ -0,0 +1,5 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker

.github/test.sh

@@ -18,6 +18,8 @@ cd upstream/manifests/install/charts
 helm install \
   --set scheduler.image=ghcr.io/flux-framework/fluence:latest \
   --set scheduler.sidecarimage=ghcr.io/flux-framework/fluence-sidecar:latest \
+  --set controller.image=ghcr.io/flux-framework/fluence-controller:latest \
+  --set controller.pullPolicy=Never \
   --set scheduler.pullPolicy=Never \
   --set scheduler.sidecarPullPolicy=Never \
     schedscheduler-plugins as-a-second-scheduler/

.github/workflows/build-deploy.yaml

@@ -18,7 +18,7 @@ jobs:
     name: build fluence
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v4
       with:
         go-version: ^1.19
 
@@ -45,7 +45,44 @@ jobs:
     - name: Deploy Container
       if: (github.event_name != 'pull_request')
       run: docker push ${{ env.container }} --all-tags
-
+
+  build-controller:
+    permissions:
+      packages: write
+    env:
+      container: ghcr.io/flux-framework/fluence-controller
+    runs-on: ubuntu-latest
+    name: build fluence-controller
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
+      with:
+        go-version: ^1.19
+
+    - name: Build Containers
+      run: |
+        make prepare
+        make build REGISTRY=ghcr.io/flux-framework CONTROLLER_IMAGE=fluence-controller
+
+    - name: Tag Release Image
+      if: (github.event_name == 'release')
+      run: |
+        tag=${GITHUB_REF#refs/tags/}
+        echo "Tagging and releasing ${{ env.container}}:${tag}"        
+        docker tag ${{ env.container }}:latest ${{ env.container }}:${tag}
+
+    - name: GHCR Login
+      if: (github.event_name != 'pull_request')
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Deploy Container
+      if: (github.event_name != 'pull_request')
+      run: docker push ${{ env.container }} --all-tags
+
   build-sidecar:
     permissions:
       packages: write
@@ -55,7 +92,7 @@ jobs:
     name: build sidecar
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v4
       with:
         go-version: ^1.19
 

.github/workflows/test.yaml

@@ -11,38 +11,50 @@ on:
 
 jobs:
   build-fluence:
+
+    # The scheduler and controller are built together with the hack script
+    # in the upstream scheduler-plugins
     env:
       container: ghcr.io/flux-framework/fluence
+      controller: ghcr.io/flux-framework/fluence-controller
     runs-on: ubuntu-latest
     name: build fluence
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v4
       with:
         go-version: ^1.19
 
     - name: Build Containers
       run: |
         make prepare
-        make build REGISTRY=ghcr.io/flux-framework SCHEDULER_IMAGE=fluence
+        make build REGISTRY=ghcr.io/flux-framework SCHEDULER_IMAGE=fluence CONTROLLER_IMAGE=fluence-controller
 
-    - name: Save Container
-      run: docker save ${{ env.container }} | gzip > fluence_latest.tar.gz
+    - name: Save Containers
+      run: |
+        docker save ${{ env.container }} | gzip > fluence_latest.tar.gz
+        docker save ${{ env.controller }} | gzip > fluence_controller_latest.tar.gz
 
     - name: Upload container artifact
       uses: actions/upload-artifact@v4
       with:
         name: fluence
         path: fluence_latest.tar.gz
-
+
+    - name: Upload container artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: fluence_controller
+        path: fluence_controller_latest.tar.gz
+
   build-sidecar:
     env:
       container: ghcr.io/flux-framework/fluence-sidecar
     runs-on: ubuntu-latest
     name: build sidecar
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v4
       with:
         go-version: ^1.19
 
@@ -59,22 +71,23 @@ jobs:
       with:
         name: fluence_sidecar
         path: fluence_sidecar_latest.tar.gz
-      
+
   test-fluence:
     needs: [build-fluence, build-sidecar]
     permissions:
       packages: write
     env:
       fluence_container: ghcr.io/flux-framework/fluence
       sidecar_container: ghcr.io/flux-framework/fluence-sidecar
+      controller_container: ghcr.io/flux-framework/fluence-controller
 
     runs-on: ubuntu-latest
-    name: build fluence
+    name: test fluence
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v4
       with:
-        go-version: ^1.20
+        go-version: ^1.19
 
     - name: Download fluence artifact
       uses: actions/download-artifact@v4
@@ -88,11 +101,27 @@ jobs:
         name: fluence_sidecar
         path: /tmp
 
+    - name: Download fluence_controller artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: fluence_controller
+        path: /tmp
+
+    - name: Make Space For Build
+      run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+
     - name: Load Docker images
       run: |
         ls /tmp/*.tar.gz
         docker load --input /tmp/fluence_sidecar_latest.tar.gz
+        rm /tmp/fluence_sidecar_latest.tar.gz
         docker load --input /tmp/fluence_latest.tar.gz
+        rm /tmp/fluence_latest.tar.gz
+        docker load --input /tmp/fluence_controller_latest.tar.gz
+        rm /tmp/fluence_controller_latest.tar.gz
         docker image ls -a | grep fluence
 
     - name: Create Kind Cluster
@@ -101,15 +130,23 @@ jobs:
         cluster_name: kind
         kubectl_version: v1.28.2
         version: v0.20.0
+        config: ./.github/test-kind-config.yaml
 
     - name: Load Docker Containers into Kind
       env:
         fluence: ${{ env.fluence_container }}
         sidecar: ${{ env.sidecar_container }}
+        controller: ${{ env.controller_container }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         kind load docker-image ${fluence}
         kind load docker-image ${sidecar}
+        kind load docker-image ${controller}
+
+    - name: Install Cert Manager
+      run: |
+        kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.1/cert-manager.yaml
+        sleep 10
 
     - name: Test Fluence
       run: /bin/bash ./.github/test.sh
@@ -122,6 +159,8 @@ jobs:
         docker tag ${{ env.fluence_container }}:latest ${{ env.fluence_container }}:${tag}
         echo "Tagging and releasing ${{ env.sidecar_container}}:${tag}"        
         docker tag ${{ env.sidecar_container }}:latest ${{ env.sidecar_container }}:${tag}
+        echo "Tagging and releasing ${{ env.controller_container}}:${tag}"        
+        docker tag ${{ env.controller_container }}:latest ${{ env.controller_container }}:${tag}
 
      # If we get here, tests pass, and we can deploy
     - name: GHCR Login
@@ -137,3 +176,4 @@ jobs:
       run: |
         docker push ${{ env.fluence_container }} --all-tags
         docker push ${{ env.sidecar_container }} --all-tags
+        docker push ${{ env.controller_container }} --all-tags

.gitignore

@@ -1,3 +1,6 @@
 plugins
 upstream
 scheduler-plugins
+sig-scheduler-plugins/pkg/fluence/bin/
+src/bin
+src/fluence/vendor

Makefile

@@ -10,24 +10,41 @@ SIDECAR_IMAGE ?= fluence-sidecar:latest
 CONTROLLER_IMAGE ?= fluence-controller
 SCHEDULER_IMAGE ?= fluence
 
-.PHONY: all build build-sidecar prepare push push-sidecar push-controller
+.PHONY: all build build-sidecar clone update push push-sidecar push-controller
 
-all: build-sidecar prepare build
+all: prepare build-sidecar build
 
 build-sidecar: 
 	make -C ./src LOCAL_REGISTRY=${REGISTRY} LOCAL_IMAGE=${SIDECAR_IMAGE}
 
-prepare: 
+clone:
 	if [ -d "$(CLONE_UPSTREAM)" ]; then echo "Upstream is cloned"; else git clone $(UPSTREAM) ./$(CLONE_UPSTREAM); fi
+
+update: clone
+	git -C $(CLONE_UPSTREAM) pull origin master
+
+prepare: clone
 	# These are entirely new directory structures
+	rm -rf $(CLONE_UPSTREAM)/pkg/fluence
+	rm -rf $(CLONE_UPSTREAM)/pkg/logger
+	# rm -rf $(CLONE_UPSTREAM)/cmd/app
+	rm -rf $(CLONE_UPSTREAM)/pkg/controllers/podgroup_controller.go
+	rm -rf $(CLONE_UPSTREAM)/cmd/controller/app/server.go
+	cp -R sig-scheduler-plugins/pkg/logger $(CLONE_UPSTREAM)/pkg/logger
 	cp -R sig-scheduler-plugins/pkg/fluence $(CLONE_UPSTREAM)/pkg/fluence
-	cp -R sig-scheduler-plugins/manifests/fluence $(CLONE_UPSTREAM)/manifests/fluence
+	cp -R sig-scheduler-plugins/pkg/controllers/* $(CLONE_UPSTREAM)/pkg/controllers/
+	# This is the one exception not from sig-scheduler-plugins because it is needed in both spots
+	cp -R src/fluence/fluxcli-grpc $(CLONE_UPSTREAM)/pkg/fluence/fluxcli-grpc
+	# cp -R sig-scheduler-plugins/cmd/app ./upstream/cmd/app
 	# These are files with subtle changes to add fluence
 	cp sig-scheduler-plugins/cmd/scheduler/main.go ./upstream/cmd/scheduler/main.go
-	cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml $(CLONE_UPSTREAM)/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml
+	cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/*.yaml $(CLONE_UPSTREAM)/manifests/install/charts/as-a-second-scheduler/templates/
+	cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/crds/*.yaml $(CLONE_UPSTREAM)/manifests/install/charts/as-a-second-scheduler/crds/
 	cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/values.yaml $(CLONE_UPSTREAM)/manifests/install/charts/as-a-second-scheduler/values.yaml
+	cp sig-scheduler-plugins/apis/scheduling/v1alpha1/*.go $(CLONE_UPSTREAM)/apis/scheduling/v1alpha1/
+	cp sig-scheduler-plugins/cmd/controller/app/server.go $(CLONE_UPSTREAM)/cmd/controller/app/server.go
 
-build:
+build: prepare
 	REGISTRY=${REGISTRY} IMAGE=${SCHEDULER_IMAGE} CONTROLLER_IMAGE=${CONTROLLER_IMAGE} $(BASH) $(CLONE_UPSTREAM)/hack/build-images.sh
 
 push-sidecar: