diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index d9f2a7051..b13f9f00c 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -156,29 +156,29 @@ To use the Docker container as an interactive virtual environment, you can run a ### Using Singularity/Apptainer instead of Docker -Since many compute clusters don't allow the usage of Docker due to securtiy concerns and instead encourage the use of [Singularity/Apptainer](https://github.com/apptainer/apptainer) (formerly Singularity, now called Apptainer), we also provide instructions on how to build an Apptainer container based on the here provided Dockerfile. - -To convert the Dockerfile into an Apptainer definition file, we will use [spython](https://github.com/singularityhub/singularity-cli): +Since many compute clusters don't allow the usage of Docker due to securtiy concerns and instead encourage the use of [Singularity/Apptainer](https://github.com/apptainer/apptainer) (formerly Singularity, now called Apptainer), we also provide an Apptainer recipe (located at `docker/Singularity.def`) that can be used to build an image by running ```bash -pip3 install spython -cd algorithmic-efficiency/docker -spython recipe Dockerfile &> Singularity.def +singularity build --fakeroot .sif Singularity.def ``` -Now we can build the Apptainer image by running - +Note that this can take several minutes. Then, to start a shell session with GPU support (by using the `--nv` flag), we can run ```bash -singularity build --fakeroot .sif Singularity.def +singularity shell --bind $HOME/data:/data,$HOME/experiment_runs:/experiment_runs \ + --nv .sif ``` -To start a shell session with GPU support (by using the `--nv` flag), we can run +Note the `--bind` flag which, similarly to Docker, allows to bind specific paths on the host system and the container, as explained [here](https://docs.sylabs.io/guides/3.7/user-guide/bind_paths_and_mounts.html). + +Also note that we generated `Singularity.def` automatically from the `Dockerfile` using [spython](https://github.com/singularityhub/singularity-cli), as follows: ```bash -singularity shell --nv .sif +pip3 install spython +cd algorithmic-efficiency/docker +python scripts/singularity_converter.py -i Dockerfile -o Singularity.def ``` -Similarly to Docker, Apptainer allows you to bind specific paths on the host system and the container by specifying the `--bind` flag, as explained [here](https://docs.sylabs.io/guides/3.7/user-guide/bind_paths_and_mounts.html). +Users that wish to customize their images are invited to check and modify the `Singularity.def` recipe and the `singularity_converter.py` script. ## Download the Data diff --git a/docker/Singularity.def b/docker/Singularity.def new file mode 100644 index 000000000..5f5c31d60 --- /dev/null +++ b/docker/Singularity.def @@ -0,0 +1,76 @@ +Bootstrap: docker +From: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 +Stage: spython-base + +%post +# Dockerfile for AlgoPerf environment. +# To build Docker image with only Jax GPU installed: +# docker build -t --build-arg framework=jax +# To build Docker image with Pytorch GPU installed: +# docker build -t --build-arg framework=pytorch + +# To build Docker image + +# Installing machine packages +echo "Setting up machine" +apt-get update +apt-get install -y curl tar +DEBIAN_FRONTEND=noninteractive apt-get install -y git python3 pip wget ffmpeg +apt-get install libtcmalloc-minimal4 +apt-get install unzip +apt-get install pigz +export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4 + +# Install GCP tools +echo "Setting up gsutil" +curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-413.0.0-linux-x86_64.tar.gz +tar -xf google-cloud-cli-413.0.0-linux-x86_64.tar.gz +yes | ./google-cloud-sdk/install.sh + +# Directory setup for input and output +echo "Setting up directories for data and experiment_runs" +mkdir -p data/ +mkdir -p experiment_runs/ + +# Install Algorithmic efficiency repo +echo "Setting up algorithmic_efficiency repo" +branch="main" +framework="both" +git_url=https://github.com/mlcommons/algorithmic-efficiency.git +git clone $git_url && cd /algorithmic-efficiency +cd /algorithmic-efficiency && git checkout $branch + +cd /algorithmic-efficiency && pip install -e '.[full]' + +if [ "$framework" = "jax" ] ; then \ +echo "Installing Jax GPU" \ +&& cd /algorithmic-efficiency \ +&& pip install -e '.[jax_gpu]' -f 'https://storage.googleapis.com/jax-releases/jax_cuda_releases.html' \ +&& pip install -e '.[pytorch_cpu]' -f 'https://download.pytorch.org/whl/torch_stable.html'; \ +elif [ "$framework" = "pytorch" ] ; then \ +echo "Installing Pytorch GPU" \ +&& cd /algorithmic-efficiency \ +&& pip install -e '.[jax_cpu]' \ +&& pip install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/torch_stable.html'; \ +elif [ "$framework" = "both" ] ; then \ +echo "Installing Jax GPU and Pytorch GPU" \ +&& cd /algorithmic-efficiency \ +&& pip install -e '.[jax_gpu]' -f 'https://storage.googleapis.com/jax-releases/jax_cuda_releases.html' \ +&& pip install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/torch_stable.html'; \ +else \ +echo "Invalid build-arg $framework: framework should be either jax, pytorch or both." >&2 \ +&& exit 1 ; \ +fi + +cd /algorithmic-efficiency && pip install -e '.[wandb]' + +cd /algorithmic-efficiency && git fetch origin +cd /algorithmic-efficiency && git pull + +# Todo: remove this, this is temporary for developing +chmod a+x /algorithmic-efficiency/docker/scripts/startup.sh + +%runscript +exec bash /algorithmic-efficiency/docker/scripts/startup.sh "$@" +%startscript +exec bash /algorithmic-efficiency/docker/scripts/startup.sh "$@" \ No newline at end of file diff --git a/docker/scripts/singularity_converter.py b/docker/scripts/singularity_converter.py new file mode 100644 index 000000000..48c521009 --- /dev/null +++ b/docker/scripts/singularity_converter.py @@ -0,0 +1,48 @@ +""" +This script is a modification of the +``spython recipe Dockerfile &> Singularity.def`` command, implemented here: +github.com/singularityhub/singularity-cli/blob/master/spython/client/recipe.py + +It converts the Docker recipe to Singularity, but suppressing any %files +command. Usage example: + +python singularity_converter.py -i Dockerfile -o Singularity.def +""" + +import argparse + +from spython.main.parse.parsers import get_parser +from spython.main.parse.writers import get_writer + +# globals +ENTRY_POINT = "/bin/bash" # seems to be a good default +FORCE = False # seems to be a good default +# +parser = argparse.ArgumentParser(description="Custom Singularity converter") +parser.add_argument( + "-i", "--input", type=str, help="Docker input path", default="Dockerfile") +parser.add_argument( + "-o", + "--output", + type=str, + help="Singularity output path", + default="Singularity.def", +) +args = parser.parse_args() +INPUT_DOCKERFILE_PATH = args.input +OUTPUT_SINGULARITY_PATH = args.output + +# create Docker parser and Singularity writer +parser = get_parser("docker") +writer = get_writer("singularity") + +# parse Dockerfile into Singularity and suppress %files commands +recipeParser = parser(INPUT_DOCKERFILE_PATH) +recipeWriter = writer(recipeParser.recipe) +(key,) = recipeParser.recipe.keys() +recipeWriter.recipe[key].files = [] + +# convert to string and save to output file +result = recipeWriter.convert(runscript=ENTRY_POINT, force=FORCE) +with open(OUTPUT_SINGULARITY_PATH, "w") as f: + f.write(result)