Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cluster setup update #814

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Cluster setup update #814

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ef7c100
fix: Setup missing service account roles
nkemnitz Oct 22, 2024
9b1db50
fix: Create zutils Artifact Registry repo
nkemnitz Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions scripts/gcp/create_corgie_igneous_cluster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,25 @@
import argparse
import subprocess

CREATE_SERVICE_ACCOUNT_TMPL = 'gcloud iam service-accounts create {CLUSTER_NAME}-worker --project={PROJECT_NAME}'
CREATE_SERVICE_ACCOUNT_TMPL = (
"gcloud iam service-accounts create {CLUSTER_NAME}-worker --project={PROJECT_NAME}"
)

CREATE_COMMAND_TMPL = 'gcloud beta container --project "{PROJECT_NAME}" clusters create "{CLUSTER_NAME}" --region "{REGION}" --no-enable-basic-auth --release-channel "regular" --machine-type "e2-medium" --image-type "COS_CONTAINERD" --disk-type "pd-standard" --disk-size "10" --metadata disable-legacy-endpoints=true --service-account "{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --max-pods-per-node "16" --spot --num-nodes "1" --logging=SYSTEM,WORKLOAD --monitoring=SYSTEM --enable-ip-alias --network "projects/{PROJECT_NAME}/global/networks/default" --subnetwork "projects/{PROJECT_NAME}/regions/{REGION}/subnetworks/default" --no-enable-intra-node-visibility --default-max-pods-per-node "16" --enable-dataplane-v2 --no-enable-master-authorized-networks --addons HorizontalPodAutoscaling,GcePersistentDiskCsiDriver --enable-autoupgrade --enable-autorepair --max-surge-upgrade 0 --max-unavailable-upgrade 1 --maintenance-window-start "2022-01-19T05:00:00Z" --maintenance-window-end "2022-01-20T05:00:00Z" --maintenance-window-recurrence "FREQ=WEEKLY;BYDAY=SA,SU" --labels owner={USERNAME} --workload-pool "{PROJECT_NAME}.svc.id.goog" --enable-shielded-nodes --node-locations {NODE_LOCATIONS} --enable-image-streaming'

ADD_CPU_COMMAND_TMPL = 'gcloud beta container --project "{PROJECT_NAME}" node-pools create "cpu-t2d-standard-16" --cluster "{CLUSTER_NAME}" --region "{REGION}" --machine-type "t2d-standard-16" --image-type "COS_CONTAINERD" --disk-type "pd-standard" --disk-size "64" --metadata disable-legacy-endpoints=true --service-account "{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --spot --enable-autoupgrade --enable-autorepair --max-surge-upgrade 1 --max-unavailable-upgrade 0 --max-pods-per-node "16" --node-locations {NODE_LOCATIONS} --enable-autoscaling --num-nodes 0 --total-min-nodes=0 --total-max-nodes=10 --node-taints=worker-pool=true:NoSchedule --enable-image-streaming'
ADD_GPU_COMMAND_TMPL = 'gcloud beta container --project "{PROJECT_NAME}" node-pools create "gpu-g2-standard-8" --cluster "{CLUSTER_NAME}" --region "{REGION}" --machine-type "g2-standard-8" --accelerator "type=nvidia-l4,count=1" --image-type "COS_CONTAINERD" --disk-type "pd-balanced" --disk-size "64" --metadata disable-legacy-endpoints=true --service-account "{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --spot --enable-autoupgrade --enable-autorepair --max-surge-upgrade 1 --max-unavailable-upgrade 0 --max-pods-per-node "16" --node-locations {NODE_LOCATIONS} --enable-autoscaling --num-nodes 0 --total-min-nodes=0 --total-max-nodes=10 --node-taints=worker-pool=true:NoSchedule --enable-image-streaming'

ADD_WORKLOAD_IDENTITY_TMPL = 'gcloud container clusters get-credentials {CLUSTER_NAME} --region {REGION} --project {PROJECT_NAME} && gcloud projects add-iam-policy-binding {PROJECT_NAME} --member "serviceAccount:{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --role "roles/storage.objectUser" && gcloud projects add-iam-policy-binding {PROJECT_NAME} --member "serviceAccount:{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --role "roles/artifactregistry.reader" && gcloud iam service-accounts add-iam-policy-binding {CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com --role roles/iam.workloadIdentityUser --member "serviceAccount:{PROJECT_NAME}.svc.id.goog[default/default]" --project {PROJECT_NAME} && kubectl annotate serviceaccount default --namespace default iam.gke.io/gcp-service-account={CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com'
ADD_WORKLOAD_IDENTITY_TMPL = 'gcloud container clusters get-credentials {CLUSTER_NAME} --region {REGION} --project {PROJECT_NAME} \
&& gcloud projects add-iam-policy-binding {PROJECT_NAME} --member "serviceAccount:{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --role "roles/storage.objectUser" \
&& gcloud projects add-iam-policy-binding {PROJECT_NAME} --member "serviceAccount:{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --role "roles/artifactregistry.reader" \
&& gcloud projects add-iam-policy-binding {PROJECT_NAME} --member "serviceAccount:[email protected]" --role "roles/container.developer" \
&& gcloud projects add-iam-policy-binding zetta-research --member "serviceAccount:{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --role "roles/storage.objectUser" \
&& gcloud projects add-iam-policy-binding zetta-research --member "serviceAccount:{CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com" --role "roles/datastore.user" \
&& gcloud iam service-accounts add-iam-policy-binding {CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com --role roles/iam.workloadIdentityUser --member "serviceAccount:{PROJECT_NAME}.svc.id.goog[default/default]" --project {PROJECT_NAME} \
&& kubectl annotate serviceaccount default --namespace default iam.gke.io/gcp-service-account={CLUSTER_NAME}-worker@{PROJECT_NAME}.iam.gserviceaccount.com'

CREATE_ARTIFACT_REGISTRY_REPO_TMPL = "gcloud artifacts repositories create zutils --repository-format=docker --location={REGION} --project={PROJECT_NAME}"

CONFIGURE_DRIVERS_COMMAND_TMPL = "gcloud container clusters get-credentials {CLUSTER_NAME} --region {REGION} --project {PROJECT_NAME}; kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded-latest.yaml"

Expand Down Expand Up @@ -62,6 +73,9 @@ def main(): # pylint: disable=too-many-statements
)
parser.add_argument("--add_cpu", action="store_true", help="Add CPU node pool.")
parser.add_argument("--add_gpu", action="store_true", help="Add GPU node pool.")
parser.add_argument(
"--add-repo", action="store_true", help="Add Artifact Registry Repository."
)

args = parser.parse_args()
# cluster_version = args.cluster_version
Expand Down Expand Up @@ -129,6 +143,13 @@ def main(): # pylint: disable=too-many-statements
print(f"Running: \n{add_wi_command}")
subprocess.call(add_wi_command, shell=True)

if args.add_repo:
create_repo_command = CREATE_ARTIFACT_REGISTRY_REPO_TMPL
create_repo_command = create_repo_command.replace("{REGION}", args.region)
create_repo_command = create_repo_command.replace("{PROJECT_NAME}", args.project_name)
print(f"Running: \n{create_repo_command}")
subprocess.call(create_repo_command, shell=True)


if __name__ == "__main__":
main()
Loading