diff --git a/src/Dockerfile b/src/Dockerfile index 6dd284c..96bec04 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -11,3 +11,8 @@ RUN groupadd -g $GID $HOST_USER && adduser --gid $GID -u $UID --gecos "" \ --disabled-password $HOST_USER && \ adduser $HOST_USER sudo && groupadd -g 1338 docker && adduser $HOST_USER docker && \ ssed -i '/^%sudo/s/ALL$/NOPASSWD:ALL/' /etc/sudoers + +# Update containerimages.conf in case of the file is modified manually. +# On slurm worker, they should be same. On controller, it might be different if user adds new images to the conf file before running setup.sh. +RUN rm /usr/local/share/slurm_gcp_docker/src/containerimages.conf +COPY containerimages.conf /usr/local/share/slurm_gcp_docker/src/ diff --git a/src/containerimages.conf b/src/containerimages.conf new file mode 100644 index 0000000..1f9e3fc --- /dev/null +++ b/src/containerimages.conf @@ -0,0 +1,7 @@ + +## List of images to be pulled to /mnt/nfs/clust_containers. Start line with "=". + +=gcr.io/broad-getzlab-workflows/cga_production_pipeline:v0.1 +=gcr.io/broad-getzlab-workflows/base_image +=chipstewart/pcawg_full:3 + diff --git a/src/prefect_config.toml b/src/prefect_config.toml new file mode 100644 index 0000000..98a7993 --- /dev/null +++ b/src/prefect_config.toml @@ -0,0 +1,12 @@ + +## This file will be read when prefect is imported + +additionalimagestores = ["/mnt/nfs/clust_containers"] +backend = "server" + +# output_dir_stem = "/mnt/nfs/workspace" +# use_podman = true +# +# [canine_backend_conf] +# type = "LocalDocker" + diff --git a/src/provision_server.py b/src/provision_server.py index 411d666..704d880 100755 --- a/src/provision_server.py +++ b/src/provision_server.py @@ -80,7 +80,7 @@ def print_conf(D, path): # Slurm conf. file cgroup.conf and boto conf can be copied-as is # (other conf. files will need editing below) subprocess.check_call( - "cp {CPR}/conf/cgroup.conf /mnt/nfs/clust_conf/slurm; cp {CPR}/conf/boto.conf /mnt/nfs/clust_conf/misc".format( + "cp {CPR}/conf/cgroup.conf /mnt/nfs/clust_conf/slurm; cp {CPR}/conf/boto.conf /mnt/nfs/clust_conf/misc; cp {CPR}/src/containerimages.conf /mnt/nfs/clust_conf/misc/containerimages.conf".format( CPR = shlex.quote(CLUST_PROV_ROOT) ), shell = True @@ -144,6 +144,13 @@ def print_conf(D, path): print_conf(C, "/mnt/nfs/clust_conf/slurm/slurmdbd.conf") + # + # Pull readonly container images. If those images already exist, it should be fast. + subprocess.check_call(""" + cat /mnt/nfs/clust_conf/misc/containerimages.conf | grep '^=' | sed 's/^=//' | xargs -L1 sudo podman --root /mnt/nfs/clust_containers pull + """, shell=True, executable = '/bin/bash' + ) + # # start Slurm controller print("Checking for running Slurm controller ... ") diff --git a/src/setup.sh b/src/setup.sh index 803e75d..814f018 100755 --- a/src/setup.sh +++ b/src/setup.sh @@ -13,6 +13,10 @@ IVERSION=$(tr . - <<< $VERSION) ./generate_container_host_image.py -i slurm-gcp-docker-${IVERSION}-`git rev-parse --short HEAD`-$USER +# copy prefect config, this will be read by prefect when it is imported +mkdir -p ~/.prefect +cp prefect_config.toml ~/.prefect/config.toml + sudo docker build -t broadinstitute/slurm_gcp_docker:$VERSION \ -t broadinstitute/slurm_gcp_docker:latest \ --build-arg HOST_USER=$USER --build-arg UID=$UID --build-arg GID=$(id -g) .