From df0fe3d019841f9d8aaa5978eae146b685339f25 Mon Sep 17 00:00:00 2001 From: Jialin Ma Date: Mon, 23 Nov 2020 07:42:32 -0500 Subject: [PATCH 1/2] Set up read-only container image store on NFS --- conf/containerimages.conf | 7 +++++++ src/prefect_config.toml | 12 ++++++++++++ src/provision_server.py | 9 ++++++++- src/setup.sh | 4 ++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 conf/containerimages.conf create mode 100644 src/prefect_config.toml diff --git a/conf/containerimages.conf b/conf/containerimages.conf new file mode 100644 index 0000000..1f9e3fc --- /dev/null +++ b/conf/containerimages.conf @@ -0,0 +1,7 @@ + +## List of images to be pulled to /mnt/nfs/clust_containers. Start line with "=". + +=gcr.io/broad-getzlab-workflows/cga_production_pipeline:v0.1 +=gcr.io/broad-getzlab-workflows/base_image +=chipstewart/pcawg_full:3 + diff --git a/src/prefect_config.toml b/src/prefect_config.toml new file mode 100644 index 0000000..98a7993 --- /dev/null +++ b/src/prefect_config.toml @@ -0,0 +1,12 @@ + +## This file will be read when prefect is imported + +additionalimagestores = ["/mnt/nfs/clust_containers"] +backend = "server" + +# output_dir_stem = "/mnt/nfs/workspace" +# use_podman = true +# +# [canine_backend_conf] +# type = "LocalDocker" + diff --git a/src/provision_server.py b/src/provision_server.py index 411d666..1cb23e4 100755 --- a/src/provision_server.py +++ b/src/provision_server.py @@ -80,7 +80,7 @@ def print_conf(D, path): # Slurm conf. file cgroup.conf and boto conf can be copied-as is # (other conf. files will need editing below) subprocess.check_call( - "cp {CPR}/conf/cgroup.conf /mnt/nfs/clust_conf/slurm; cp {CPR}/conf/boto.conf /mnt/nfs/clust_conf/misc".format( + "cp {CPR}/conf/cgroup.conf /mnt/nfs/clust_conf/slurm; cp {CPR}/conf/boto.conf /mnt/nfs/clust_conf/misc; cp {CPR}/conf/containerimages.conf /mnt/nfs/clust_conf/misc/containerimages.conf".format( CPR = shlex.quote(CLUST_PROV_ROOT) ), shell = True @@ -144,6 +144,13 @@ def print_conf(D, path): print_conf(C, "/mnt/nfs/clust_conf/slurm/slurmdbd.conf") + # + # Pull readonly container images. If those images already exist, it should be fast. + subprocess.check_call(""" + cat /mnt/nfs/clust_conf/misc/containerimages.conf | grep '^=' | sed 's/^=//' | xargs -L1 sudo podman --root /mnt/nfs/clust_containers pull + """, shell=True, executable = '/bin/bash' + ) + # # start Slurm controller print("Checking for running Slurm controller ... ") diff --git a/src/setup.sh b/src/setup.sh index 803e75d..814f018 100755 --- a/src/setup.sh +++ b/src/setup.sh @@ -13,6 +13,10 @@ IVERSION=$(tr . - <<< $VERSION) ./generate_container_host_image.py -i slurm-gcp-docker-${IVERSION}-`git rev-parse --short HEAD`-$USER +# copy prefect config, this will be read by prefect when it is imported +mkdir -p ~/.prefect +cp prefect_config.toml ~/.prefect/config.toml + sudo docker build -t broadinstitute/slurm_gcp_docker:$VERSION \ -t broadinstitute/slurm_gcp_docker:latest \ --build-arg HOST_USER=$USER --build-arg UID=$UID --build-arg GID=$(id -g) . From 3f13dacb7fdde5c0190b7d75da9f61227ab3f547 Mon Sep 17 00:00:00 2001 From: Jialin Ma Date: Mon, 23 Nov 2020 08:19:27 -0500 Subject: [PATCH 2/2] Move containerimages.conf to src/ and update it in Dockerfile --- src/Dockerfile | 5 +++++ {conf => src}/containerimages.conf | 0 src/provision_server.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) rename {conf => src}/containerimages.conf (100%) diff --git a/src/Dockerfile b/src/Dockerfile index 6dd284c..96bec04 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -11,3 +11,8 @@ RUN groupadd -g $GID $HOST_USER && adduser --gid $GID -u $UID --gecos "" \ --disabled-password $HOST_USER && \ adduser $HOST_USER sudo && groupadd -g 1338 docker && adduser $HOST_USER docker && \ ssed -i '/^%sudo/s/ALL$/NOPASSWD:ALL/' /etc/sudoers + +# Update containerimages.conf in case of the file is modified manually. +# On slurm worker, they should be same. On controller, it might be different if user adds new images to the conf file before running setup.sh. +RUN rm /usr/local/share/slurm_gcp_docker/src/containerimages.conf +COPY containerimages.conf /usr/local/share/slurm_gcp_docker/src/ diff --git a/conf/containerimages.conf b/src/containerimages.conf similarity index 100% rename from conf/containerimages.conf rename to src/containerimages.conf diff --git a/src/provision_server.py b/src/provision_server.py index 1cb23e4..704d880 100755 --- a/src/provision_server.py +++ b/src/provision_server.py @@ -80,7 +80,7 @@ def print_conf(D, path): # Slurm conf. file cgroup.conf and boto conf can be copied-as is # (other conf. files will need editing below) subprocess.check_call( - "cp {CPR}/conf/cgroup.conf /mnt/nfs/clust_conf/slurm; cp {CPR}/conf/boto.conf /mnt/nfs/clust_conf/misc; cp {CPR}/conf/containerimages.conf /mnt/nfs/clust_conf/misc/containerimages.conf".format( + "cp {CPR}/conf/cgroup.conf /mnt/nfs/clust_conf/slurm; cp {CPR}/conf/boto.conf /mnt/nfs/clust_conf/misc; cp {CPR}/src/containerimages.conf /mnt/nfs/clust_conf/misc/containerimages.conf".format( CPR = shlex.quote(CLUST_PROV_ROOT) ), shell = True