diff --git a/.build/10activate-conda-env.sh b/.build/10activate-conda-env.sh new file mode 100755 index 0000000..ed7347f --- /dev/null +++ b/.build/10activate-conda-env.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# This registers the initialization code for the conda shell code +# It also activates default environment in the end, so we don't need to activate it manually +# Documentation: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html +eval "$(conda shell.bash hook)" diff --git a/.build/Dockerfile b/.build/Dockerfile index 1d45e7b..d135732 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -1,7 +1,7 @@ -# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'src/' +# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'custom/' # **Please do not change this file directly!** -# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'src/Dockerfile.usefulpackages'. +# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'custom/usefulpackages.Dockerfile'. # More information can be found in the README under configuration. @@ -39,12 +39,12 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for notebook server that starts but lacks all -# features (e.g., download as all possible file formats) +# Install all OS dependencies for the Server that starts +# but lacks all features (e.g., download as all possible file formats) ENV DEBIAN_FRONTEND noninteractive RUN apt-get update --yes && \ - # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as - # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) + # - `apt-get upgrade` is run to patch known vulnerabilities in apt-get packages as + # the Ubuntu base image is rebuilt too seldom sometimes (less than once a month) apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends \ # - bzip2 is necessary to extract the micromamba executable. @@ -80,36 +80,37 @@ RUN chmod a+rx /usr/local/bin/fix-permissions # Enable prompt color in the skeleton .bashrc before creating the default NB_USER # hadolint ignore=SC2016 RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ - # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 - echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + # More information in: https://github.com/jupyter/docker-stacks/pull/2047 + # and docs: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html + echo 'eval "$(conda shell.bash hook)"' >> /etc/skel/.bashrc # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group # and make sure these dirs are writable by the `users` group. RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ - useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ + useradd --no-log-init --create-home --shell /bin/bash --uid "${NB_UID}" --no-user-group "${NB_USER}" && \ mkdir -p "${CONDA_DIR}" && \ chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ chmod g+w /etc/passwd && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" USER ${NB_UID} -# Pin python version here, or set it to "default" +# Pin the Python version here, or set it to "default" ARG PYTHON_VERSION=3.11 # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ fix-permissions "/home/${NB_USER}" -# Download and install Micromamba, and initialize Conda prefix. +# Download and install Micromamba, and initialize the Conda prefix. # # Similar projects using Micromamba: # - Micromamba-Docker: # - repo2docker: -# Install Python, Mamba and jupyter_core +# Install Python, Mamba, and jupyter_core # Cleanup temporary files and remove Micromamba # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the @@ -148,7 +149,15 @@ ENTRYPOINT ["tini", "-g", "--"] CMD ["start.sh"] # Copy local files as late as possible to avoid cache busting -COPY start.sh /usr/local/bin/ +COPY run-hooks.sh start.sh /usr/local/bin/ + +USER root + +# Create dirs for startup hooks +RUN mkdir /usr/local/bin/start-notebook.d && \ + mkdir /usr/local/bin/before-notebook.d + +COPY 10activate-conda-env.sh /usr/local/bin/before-notebook.d/ # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} @@ -161,6 +170,7 @@ WORKDIR "${HOME}" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter LABEL maintainer="Jupyter Project " @@ -171,35 +181,35 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for notebook server that starts but lacks all +# Install all OS dependencies for the Server that starts but lacks all # features (e.g., download as all possible file formats) RUN apt-get update --yes && \ apt-get install --yes --no-install-recommends \ fonts-liberation \ # - pandoc is used to convert notebooks to html files - # it's not present in aarch64 ubuntu image, so we install it here + # it's not present in the aarch64 Ubuntu image, so we install it here pandoc \ # - run-one - a wrapper script that runs no more # than one unique instance of some command with a unique set of arguments, - # we use `run-one-constantly` to support `RESTARTABLE` option + # we use `run-one-constantly` to support the `RESTARTABLE` option run-one && \ apt-get clean && rm -rf /var/lib/apt/lists/* USER ${NB_UID} -# Install Jupyter Notebook, Lab, and Hub -# Generate a notebook server config +# Install JupyterLab, Jupyter Notebook, JupyterHub and NBClassic +# Generate a Jupyter Server config # Cleanup temporary files # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change WORKDIR /tmp RUN mamba install --yes \ + 'jupyterlab' \ 'notebook' \ 'jupyterhub' \ - 'jupyterlab' \ 'nbclassic' && \ - jupyter notebook --generate-config && \ + jupyter server --generate-config && \ mamba clean --all -f -y && \ npm cache clean --force && \ jupyter lab clean && \ @@ -211,23 +221,18 @@ ENV JUPYTER_PORT=8888 EXPOSE $JUPYTER_PORT # Configure container startup -CMD ["start-notebook.sh"] +CMD ["start-notebook.py"] # Copy local files as late as possible to avoid cache busting -COPY start-notebook.sh start-singleuser.sh /usr/local/bin/ -# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab +COPY start-notebook.py start-notebook.sh start-singleuser.py start-singleuser.sh /usr/local/bin/ COPY jupyter_server_config.py docker_healthcheck.py /etc/jupyter/ # Fix permissions on /etc/jupyter as root USER root - -# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205) -RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \ - /etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \ - fix-permissions /etc/jupyter/ +RUN fix-permissions /etc/jupyter/ # HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck -# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server` and `retro` jupyter commands +# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands # https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ CMD /etc/jupyter/docker_healthcheck.py || exit 1 @@ -243,6 +248,7 @@ WORKDIR "${HOME}" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter LABEL maintainer="Jupyter Project " @@ -253,10 +259,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for fully functional notebook server +# Install all OS dependencies for a fully functional Server RUN apt-get update --yes && \ apt-get install --yes --no-install-recommends \ # Common useful utilities + curl \ git \ nano-tiny \ tzdata \ @@ -282,7 +289,7 @@ RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10 # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} -# Add R mimetype option to specify how the plot returns from R to the browser +# Add an R mimetype option to specify how the plot returns from R to the browser COPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/ # Add setup scripts that may be used by downstream images or inherited images @@ -294,6 +301,7 @@ COPY setup-scripts/ /opt/setup-scripts/ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter LABEL maintainer="Jupyter Project " @@ -353,17 +361,15 @@ RUN mamba install --yes \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Install facets which does not have a pip or conda package at the moment +# Install facets package which does not have a `pip` or `conda-forge` package at the moment WORKDIR /tmp -RUN git clone https://github.com/PAIR-code/facets.git && \ +RUN git clone https://github.com/PAIR-code/facets && \ jupyter nbclassic-extension install facets/facets-dist/ --sys-prefix && \ rm -rf /tmp/facets && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Import matplotlib the first time to build the font cache. -ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" - +# Import matplotlib the first time to build the font cache RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ fix-permissions "/home/${NB_USER}" @@ -381,12 +387,6 @@ LABEL maintainer="Christoph Schranz , Mat # https://www.tensorflow.org/install/source#gpu # installation via conda leads to errors in version 4.8.2 # Install CUDA-specific nvidia libraries and update libcudnn8 before that -USER root -RUN apt-get update && \ - apt-get install -y --no-install-recommends --allow-change-held-packages libcudnn8 && \ - apt-get install -y --no-install-recommends libnvinfer-dev libnvinfer-plugin-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* -RUN cd /usr/lib/x86_64-linux-gnu && ln -s libnvinfer_plugin.so.8 libnvinfer_plugin.so.7 && ln -s libnvinfer.so.8 libnvinfer.so.7 USER ${NB_UID} RUN pip install --upgrade pip && \ pip install --no-cache-dir tensorflow==2.15.0 keras==2.15.0 && \ diff --git a/.build/docker-stacks b/.build/docker-stacks index b8d617d..1494233 160000 --- a/.build/docker-stacks +++ b/.build/docker-stacks @@ -1 +1 @@ -Subproject commit b8d617dc0568d60f6583c42f989da51ec80e9af6 +Subproject commit 1494233e27cdc70e3766ea2518e7153ee425fc4f diff --git a/.build/docker_healthcheck.py b/.build/docker_healthcheck.py index 7c35a6b..8f3338e 100755 --- a/.build/docker_healthcheck.py +++ b/.build/docker_healthcheck.py @@ -7,7 +7,7 @@ import requests -# A number of operations below deliberately don't check for possible errors +# Several operations below deliberately don't check for possible errors # As this is a healthcheck, it should succeed or raise an exception on error runtime_dir = Path("/home/") / os.environ["NB_USER"] / ".local/share/jupyter/runtime/" @@ -16,6 +16,11 @@ url = json.loads(json_file.read_bytes())["url"] url = url + "api" -r = requests.get(url, verify=False) # request without SSL verification +proxies = { + "http": "", + "https": "", +} + +r = requests.get(url, proxies=proxies, verify=False) # request without SSL verification r.raise_for_status() print(r.content) diff --git a/.build/fix-permissions b/.build/fix-permissions index d167578..47b6d0e 100755 --- a/.build/fix-permissions +++ b/.build/fix-permissions @@ -1,16 +1,14 @@ #!/bin/bash -# set permissions on a directory -# after any installation, if a directory needs to be (human) user-writable, -# run this script on it. -# It will make everything in the directory owned by the group ${NB_GID} -# and writable by that group. +# Set permissions on a directory +# After any installation, if a directory needs to be (human) user-writable, run this script on it. +# It will make everything in the directory owned by the group ${NB_GID} and writable by that group. # Deployments that want to set a specific user id can preserve permissions # by adding the `--group-add users` line to `docker run`. -# uses find to avoid touching files that already have the right permissions, -# which would cause massive image explosion +# Uses find to avoid touching files that already have the right permissions, +# which would cause a massive image explosion -# right permissions are: +# Right permissions are: # group=${NB_GID} # AND permissions include group rwX (directory-execute) # AND directories have setuid,setgid bits set diff --git a/.build/jupyter_server_config.py b/.build/jupyter_server_config.py index 679f96b..c0cca3a 100755 --- a/.build/jupyter_server_config.py +++ b/.build/jupyter_server_config.py @@ -4,6 +4,7 @@ import os import stat import subprocess +from pathlib import Path from jupyter_core.paths import jupyter_data_dir @@ -24,17 +25,16 @@ [req_distinguished_name] """ if "GEN_CERT" in os.environ: - dir_name = jupyter_data_dir() - pem_file = os.path.join(dir_name, "notebook.pem") - os.makedirs(dir_name, exist_ok=True) + dir_name = Path(jupyter_data_dir()) + dir_name.mkdir(parents=True, exist_ok=True) + pem_file = dir_name / "notebook.pem" # Generate an openssl.cnf file to set the distinguished name - cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf") - if not os.path.isfile(cnf_file): - with open(cnf_file, "w") as fh: - fh.write(OPENSSL_CONFIG) + cnf_file = Path(os.getenv("CONDA_DIR", "/usr/lib")) / "ssl/openssl.cnf" + if not cnf_file.exists(): + cnf_file.write_text(OPENSSL_CONFIG) - # Generate a certificate if one doesn't exist on disk + # Generate a certificate if one doesn't exist on a disk subprocess.check_call( [ "openssl", @@ -50,10 +50,9 @@ ] ) # Restrict access to the file - os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) - c.ServerApp.certfile = pem_file + pem_file.chmod(stat.S_IRUSR | stat.S_IWUSR) + c.ServerApp.certfile = str(pem_file) -# Change default umask for all subprocesses of the notebook server if set in -# the environment +# Change default umask for all subprocesses of the Server if set in the environment if "NB_UMASK" in os.environ: os.umask(int(os.environ["NB_UMASK"], 8)) diff --git a/.build/run-hooks.sh b/.build/run-hooks.sh new file mode 100755 index 0000000..15df23c --- /dev/null +++ b/.build/run-hooks.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# The run-hooks.sh script looks for *.sh scripts to source +# and executable files to run within a passed directory + +if [ "$#" -ne 1 ]; then + echo "Should pass exactly one directory" + return 1 +fi + +if [[ ! -d "${1}" ]]; then + echo "Directory ${1} doesn't exist or is not a directory" + return 1 +fi + +echo "Running hooks in: ${1} as uid: $(id -u) gid: $(id -g)" +for f in "${1}/"*; do + # Handling a case when the directory is empty + [ -e "${f}" ] || continue + case "${f}" in + *.sh) + echo "Sourcing shell script: ${f}" + # shellcheck disable=SC1090 + source "${f}" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "${f} has failed, continuing execution" + fi + ;; + *) + if [ -x "${f}" ]; then + echo "Running executable: ${f}" + "${f}" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "${f} has failed, continuing execution" + fi + else + echo "Ignoring non-executable: ${f}" + fi + ;; + esac +done +echo "Done running hooks in: ${1}" diff --git a/.build/setup-scripts/activate_notebook_custom_env.py b/.build/setup-scripts/activate_notebook_custom_env.py new file mode 100755 index 0000000..4d5da9b --- /dev/null +++ b/.build/setup-scripts/activate_notebook_custom_env.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import json +import os +import sys +from pathlib import Path + +env_name = sys.argv[1] +CONDA_DIR = os.environ["CONDA_DIR"] + +file = Path.home() / f".local/share/jupyter/kernels/{env_name}/kernel.json" +content = json.loads(file.read_text()) +content["env"] = { + "XML_CATALOG_FILES": "", + "PATH": f"{CONDA_DIR}/envs/{env_name}/bin:$PATH", + "CONDA_PREFIX": f"{CONDA_DIR}/envs/{env_name}", + "CONDA_PROMPT_MODIFIER": f"({env_name}) ", + "CONDA_SHLVL": "2", + "CONDA_DEFAULT_ENV": env_name, + "CONDA_PREFIX_1": CONDA_DIR, +} + +file.write_text(json.dumps(content, indent=1)) diff --git a/.build/setup-scripts/setup-julia-packages.bash b/.build/setup-scripts/setup-julia-packages.bash index faeee01..fa1421e 100755 --- a/.build/setup-scripts/setup-julia-packages.bash +++ b/.build/setup-scripts/setup-julia-packages.bash @@ -1,10 +1,32 @@ #!/bin/bash set -exuo pipefail # Requirements: -# - Run as non-root user +# - Run as a non-root user # - The JULIA_PKGDIR environment variable is set # - Julia is already set up, with the setup-julia.bash command + +# If we don't specify what CPUs the precompilation should be done for, it's +# *only* done for the target of the host doing the compilation. When the +# container runs on a host that's the same architecture, but a *different* +# generation of CPU than what the build host was, the precompilation is useless +# and Julia takes a long long time to start up. This specific multitarget comes +# from https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L20-L76, +# and may need to be updated as new CPU generations come out. +# If the architecture the container runs on is different, +# precompilation may still have to be re-done on first startup - but this +# *should* catch most of the issues. See +# https://github.com/jupyter/docker-stacks/issues/2015 for more information +if [ "$(uname -m)" == "x86_64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L24 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" +elif [ "$(uname -m)" == "aarch64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L54 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel" +fi + # Install base Julia packages julia -e ' import Pkg; @@ -17,9 +39,9 @@ Pkg.add([ Pkg.precompile(); ' -# Move the kernelspec out to the system share location. Avoids -# problems with runtime UID change not taking effect properly on the -# .local folder in the jovyan home dir. move kernelspec out of home +# Move the kernelspec out of ${HOME} to the system share location. +# Avoids problems with runtime UID change not taking effect properly +# on the .local folder in the jovyan home dir. mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" chmod -R go+rx "${CONDA_DIR}/share/jupyter" rm -rf "${HOME}/.local" diff --git a/.build/setup-scripts/setup_julia.py b/.build/setup-scripts/setup_julia.py new file mode 100755 index 0000000..0cdbe0c --- /dev/null +++ b/.build/setup-scripts/setup_julia.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Requirements: +# - Run as the root user +# - The JULIA_PKGDIR environment variable is set + +import os +import platform +import shutil +import subprocess +from pathlib import Path + +import requests + + +def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ + return { + "aarch64": "aarch64", + "arm64": "aarch64", + "x86_64": "x86_64", + }[platform] + + +def get_latest_julia_url() -> tuple[str, str]: + """ + Get the last stable version of Julia + Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813 + """ + + versions = requests.get( + "https://julialang-s3.julialang.org/bin/versions.json" + ).json() + stable_versions = {k: v for k, v in versions.items() if v["stable"]} + latest_version_files = stable_versions[max(stable_versions)]["files"] + triplet = unify_aarch64(platform.machine()) + "-linux-gnu" + file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] + return file_info["url"], file_info["version"] + + +def download_julia(julia_url: str) -> None: + """ + Downloads and unpacks julia + The resulting julia directory is "/opt/julia-VERSION/" + """ + tmp_file = Path("/tmp/julia.tar.gz") + subprocess.check_call( + ["curl", "--progress-bar", "--location", "--output", tmp_file, julia_url] + ) + shutil.unpack_archive(tmp_file, "/opt/") + tmp_file.unlink() + + +def prepare_julia(julia_version: str) -> None: + """ + Creates /usr/local/bin/julia symlink + Make Julia aware of conda libraries + Creates a directory for Julia user libraries + """ + # Link Julia installed version to /usr/local/bin, so julia launches it + subprocess.check_call( + ["ln", "-fs", f"/opt/julia-{julia_version}/bin/julia", "/usr/local/bin/julia"] + ) + + # Tell Julia where conda libraries are + Path("/etc/julia").mkdir() + Path("/etc/julia/juliarc.jl").write_text( + f'push!(Libdl.DL_LOAD_PATH, "{os.environ["CONDA_DIR"]}/lib")\n' + ) + + # Create JULIA_PKGDIR, where user libraries are installed + JULIA_PKGDIR = Path(os.environ["JULIA_PKGDIR"]) + JULIA_PKGDIR.mkdir() + subprocess.check_call(["chown", os.environ["NB_USER"], JULIA_PKGDIR]) + subprocess.check_call(["fix-permissions", JULIA_PKGDIR]) + + +if __name__ == "__main__": + julia_url, julia_version = get_latest_julia_url() + download_julia(julia_url=julia_url) + prepare_julia(julia_version=julia_version) diff --git a/.build/start-notebook.py b/.build/start-notebook.py new file mode 100755 index 0000000..b99ff31 --- /dev/null +++ b/.build/start-notebook.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shlex +import sys + +# If we are in a JupyterHub, we pass on to `start-singleuser.py` instead so it does the right thing +if "JUPYTERHUB_API_TOKEN" in os.environ: + print( + "WARNING: using start-singleuser.py instead of start-notebook.py to start a server associated with JupyterHub." + ) + command = ["/usr/local/bin/start-singleuser.py"] + sys.argv[1:] + os.execvp(command[0], command) + + +# Wrap everything in start.sh, no matter what +command = ["/usr/local/bin/start.sh"] + +# If we want to survive restarts, tell that to start.sh +if os.environ.get("RESTARTABLE") == "yes": + command.append("run-one-constantly") + +# We always launch a jupyter subcommand from this script +command.append("jupyter") + +# Launch the configured subcommand. Note that this should be a single string, so we don't split it +# We default to lab +jupyter_command = os.environ.get("DOCKER_STACKS_JUPYTER_CMD", "lab") +command.append(jupyter_command) + +# Append any optional NOTEBOOK_ARGS we were passed in. This is supposed to be multiple args passed +# on to the notebook command, so we split it correctly with shlex +if "NOTEBOOK_ARGS" in os.environ: + command += shlex.split(os.environ["NOTEBOOK_ARGS"]) + +# Pass through any other args we were passed on the command line +command += sys.argv[1:] + +# Execute the command! +os.execvp(command[0], command) diff --git a/.build/start-notebook.sh b/.build/start-notebook.sh index 4f673d2..c47ebba 100755 --- a/.build/start-notebook.sh +++ b/.build/start-notebook.sh @@ -1,22 +1,5 @@ #!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. +# Shim to emit warning and call start-notebook.py +echo "WARNING: Use start-notebook.py instead" -set -e - -# The Jupyter command to launch -# JupyterLab by default -DOCKER_STACKS_JUPYTER_CMD="${DOCKER_STACKS_JUPYTER_CMD:=lab}" - -if [[ -n "${JUPYTERHUB_API_TOKEN}" ]]; then - echo "WARNING: using start-singleuser.sh instead of start-notebook.sh to start a server associated with JupyterHub." - exec /usr/local/bin/start-singleuser.sh "$@" -fi - -wrapper="" -if [[ "${RESTARTABLE}" == "yes" ]]; then - wrapper="run-one-constantly" -fi - -# shellcheck disable=SC1091,SC2086 -exec /usr/local/bin/start.sh ${wrapper} jupyter ${DOCKER_STACKS_JUPYTER_CMD} ${NOTEBOOK_ARGS} "$@" +exec /usr/local/bin/start-notebook.py "$@" diff --git a/.build/start-singleuser.py b/.build/start-singleuser.py new file mode 100755 index 0000000..2dcf6c0 --- /dev/null +++ b/.build/start-singleuser.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shlex +import sys + +command = ["/usr/local/bin/start.sh", "jupyterhub-singleuser"] + +# set default ip to 0.0.0.0 +if "--ip=" not in os.environ.get("NOTEBOOK_ARGS", ""): + command.append("--ip=0.0.0.0") + +# Append any optional NOTEBOOK_ARGS we were passed in. This is supposed to be multiple args passed +# on to the notebook command, so we split it correctly with shlex +if "NOTEBOOK_ARGS" in os.environ: + command += shlex.split(os.environ["NOTEBOOK_ARGS"]) + +# Pass any other args we have been passed through +command += sys.argv[1:] + +# Execute the command! +os.execvp(command[0], command) diff --git a/.build/start-singleuser.sh b/.build/start-singleuser.sh index a2166e2..ecf0e06 100755 --- a/.build/start-singleuser.sh +++ b/.build/start-singleuser.sh @@ -1,13 +1,5 @@ #!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. +# Shim to emit warning and call start-singleuser.py +echo "WARNING: Use start-singleuser.py instead" -set -e - -# set default ip to 0.0.0.0 -if [[ "${NOTEBOOK_ARGS} $*" != *"--ip="* ]]; then - NOTEBOOK_ARGS="--ip=0.0.0.0 ${NOTEBOOK_ARGS}" -fi - -# shellcheck disable=SC1091,SC2086 -. /usr/local/bin/start.sh jupyterhub-singleuser ${NOTEBOOK_ARGS} "$@" +exec /usr/local/bin/start-singleuser.py "$@" diff --git a/.build/start.sh b/.build/start.sh index 7c5859e..76419b6 100755 --- a/.build/start.sh +++ b/.build/start.sh @@ -4,9 +4,9 @@ set -e -# The _log function is used for everything this script wants to log. It will -# always log errors and warnings, but can be silenced for other messages -# by setting JUPYTER_DOCKER_STACKS_QUIET environment variable. +# The _log function is used for everything this script wants to log. +# It will always log errors and warnings but can be silenced for other messages +# by setting the JUPYTER_DOCKER_STACKS_QUIET environment variable. _log () { if [[ "$*" == "ERROR:"* ]] || [[ "$*" == "WARNING:"* ]] || [[ "${JUPYTER_DOCKER_STACKS_QUIET}" == "" ]]; then echo "$@" @@ -14,39 +14,12 @@ _log () { } _log "Entered start.sh with args:" "$@" -# The run-hooks function looks for .sh scripts to source and executable files to -# run within a passed directory. -run-hooks () { - if [[ ! -d "${1}" ]] ; then - return - fi - _log "${0}: running hooks in ${1} as uid / gid: $(id -u) / $(id -g)" - for f in "${1}/"*; do - case "${f}" in - *.sh) - _log "${0}: running script ${f}" - # shellcheck disable=SC1090 - source "${f}" - ;; - *) - if [[ -x "${f}" ]] ; then - _log "${0}: running executable ${f}" - "${f}" - else - _log "${0}: ignoring non-executable ${f}" - fi - ;; - esac - done - _log "${0}: done running hooks in ${1}" -} - # A helper function to unset env vars listed in the value of the env var # JUPYTER_ENV_VARS_TO_UNSET. unset_explicit_env_vars () { if [ -n "${JUPYTER_ENV_VARS_TO_UNSET}" ]; then for env_var_to_unset in $(echo "${JUPYTER_ENV_VARS_TO_UNSET}" | tr ',' ' '); do - echo "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" + _log "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" unset "${env_var_to_unset}" done unset JUPYTER_ENV_VARS_TO_UNSET @@ -62,14 +35,15 @@ else fi # NOTE: This hook will run as the user the container was started with! -run-hooks /usr/local/bin/start-notebook.d +# shellcheck disable=SC1091 +source /usr/local/bin/run-hooks.sh /usr/local/bin/start-notebook.d # If the container started as the root user, then we have permission to refit # the jovyan user, and ensure file permissions, grant sudo rights, and such # things before we run the command passed to start.sh as the desired user # (NB_USER). # -if [ "$(id -u)" == 0 ] ; then +if [ "$(id -u)" == 0 ]; then # Environment variables: # - NB_USER: the desired username and associated home folder # - NB_UID: the desired user id @@ -77,18 +51,18 @@ if [ "$(id -u)" == 0 ] ; then # - NB_GROUP: a group name we want for the group # - GRANT_SUDO: a boolean ("1" or "yes") to grant the user sudo rights # - CHOWN_HOME: a boolean ("1" or "yes") to chown the user's home folder - # - CHOWN_EXTRA: a comma separated list of paths to chown + # - CHOWN_EXTRA: a comma-separated list of paths to chown # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands - # Refit the jovyan user to the desired the user (NB_USER) - if id jovyan &> /dev/null ; then + # Refit the jovyan user to the desired user (NB_USER) + if id jovyan &> /dev/null; then if ! usermod --home "/home/${NB_USER}" --login "${NB_USER}" jovyan 2>&1 | grep "no changes" > /dev/null; then _log "Updated the jovyan user:" _log "- username: jovyan -> ${NB_USER}" _log "- home dir: /home/jovyan -> /home/${NB_USER}" fi elif ! id -u "${NB_USER}" &> /dev/null; then - _log "ERROR: Neither the jovyan user or '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." + _log "ERROR: Neither the jovyan user nor '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." exit 1 fi # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is @@ -101,10 +75,10 @@ if [ "$(id -u)" == 0 ] ; then fi # Recreate the desired user as we want it userdel "${NB_USER}" - useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}" + useradd --no-log-init --home "/home/${NB_USER}" --shell /bin/bash --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 "${NB_USER}" fi - # Move or symlink the jovyan home directory to the desired users home + # Move or symlink the jovyan home directory to the desired user's home # directory if it doesn't already exist, and update the current working # directory to the new location if needed. if [[ "${NB_USER}" != "jovyan" ]]; then @@ -132,7 +106,7 @@ if [ "$(id -u)" == 0 ] ; then fi fi - # Optionally ensure the desired user get filesystem ownership of it's home + # Optionally ensure the desired user gets filesystem ownership of its home # folder and/or additional folders if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then _log "Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}" @@ -147,9 +121,6 @@ if [ "$(id -u)" == 0 ] ; then done fi - # Update potentially outdated environment variables since image build - export XDG_CACHE_HOME="/home/${NB_USER}/.cache" - # Prepend ${CONDA_DIR}/bin to sudo secure_path sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"${CONDA_DIR}/bin:\1\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path @@ -160,11 +131,13 @@ if [ "$(id -u)" == 0 ] ; then fi # NOTE: This hook is run as the root user! - run-hooks /usr/local/bin/before-notebook.d - + # shellcheck disable=SC1091 + source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d unset_explicit_env_vars + _log "Running as ${NB_USER}:" "${cmd[@]}" exec sudo --preserve-env --set-home --user "${NB_USER}" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" \ PATH="${PATH}" \ PYTHONPATH="${PYTHONPATH:-}" \ "${cmd[@]}" @@ -178,7 +151,7 @@ if [ "$(id -u)" == 0 ] ; then # command. The behavior can be inspected with `sudo -V` run as root. # # ref: `man sudo` https://linux.die.net/man/8/sudo - # ref: `man sudoers` https://www.sudo.ws/man/1.8.15/sudoers.man.html + # ref: `man sudoers` https://www.sudo.ws/docs/man/sudoers.man/ # # - We use the `--preserve-env` flag to pass through most environment # variables, but understand that exceptions are caused by the sudoers @@ -187,10 +160,10 @@ if [ "$(id -u)" == 0 ] ; then # - We use the `--set-home` flag to set the HOME variable appropriately. # # - To reduce the default list of variables deleted by sudo, we could have - # used `env_delete` from /etc/sudoers. It has higher priority than the + # used `env_delete` from /etc/sudoers. It has a higher priority than the # `--preserve-env` flag and the `env_keep` configuration. # - # - We preserve PATH and PYTHONPATH explicitly. Note however that sudo + # - We preserve LD_LIBRARY_PATH, PATH and PYTHONPATH explicitly. Note however that sudo # resolves `${cmd[@]}` using the "secure_path" variable we modified # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant @@ -210,7 +183,7 @@ else # Attempt to ensure the user uid we currently run as has a named entry in # the /etc/passwd file, as it avoids software crashing on hard assumptions # on such entry. Writing to the /etc/passwd was allowed for the root group - # from the Dockerfile during build. + # from the Dockerfile during the build. # # ref: https://github.com/jupyter/docker-stacks/issues/552 if ! whoami &> /dev/null; then @@ -255,8 +228,10 @@ else fi # NOTE: This hook is run as the user we started the container as! - run-hooks /usr/local/bin/before-notebook.d + # shellcheck disable=SC1091 + source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d unset_explicit_env_vars + _log "Executing the command:" "${cmd[@]}" exec "${cmd[@]}" fi diff --git a/README.md b/README.md index 057387c..7a92da5 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,9 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc ```bash cd your-working-directory ll data # this path will be mounted by default - docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.5_cuda-12.0_ubuntu-22.04_python-only + docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-12.0_ubuntu-22.04 ``` - This starts an instance of *GPU-Jupyter* with the tag `v1.5_cuda-12.0_ubuntu-22.04_python-only` at [http://localhost:8848](http://localhost:8848) (port `8848`). + This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-12.0_ubuntu-22.04` at [http://localhost:8848](http://localhost:8848) (port `8848`). To log into Jupyterlab, you have to specify a token that you get from: ```bash docker exec -it [container-ID/name] jupyter server list @@ -82,6 +82,15 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc Additionally, data within the host's `data` directory is shared with the container. Note that the following images of GPU-Jupyter are available on [Dockerhub](https://hub.docker.com/r/cschranz/gpu-jupyter): + - `v1.6_cuda-12.0_ubuntu-22.04` (full image) + - `v1.6_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.6_cuda-12.0_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) + - `v1.6_cuda-11.8_ubuntu-22.04` (full image) + - `v1.6_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.6_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) + - `v1.6_cuda-11.6_ubuntu-20.04` (full image) + - `v1.6_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.6_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - `v1.5_cuda-12.0_ubuntu-22.04` (full image) - `v1.5_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - `v1.5_cuda-12.0_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) @@ -107,7 +116,7 @@ Additionally, data within the host's `data` directory is shared with the contain - `v1.4_cuda-10.1_ubuntu-18.04_python-only` (only with a python interpreter and without Julia and R) - `v1.4_cuda-10.1_ubuntu-18.04_slim` (only with a python interpreter and without additional packages) - The version, e.g. `v1.5`, declares the version of the generator setup. + The version, e.g. `v1.6`, declares the version of the generator setup. The Cuda version, e.g. `cuda-12.0`, must match the CUDA driver version and be supported by the GPU libraries. These and older versions of GPU-Jupyter are listed on [Dockerhub](https://hub.docker.com/r/cschranz/gpu-jupyter/tags?page=1&ordering=last_updated). In case you are using another version or the GPU libraries don't work on your hardware, please try to build the image on your own as described below. @@ -122,7 +131,7 @@ we recommend checking out this [tutorial](https://www.youtube.com/watch?v=7wfPqA ## Build Your Image -Building a custom Docker image is the recommended option if you have a different GPU architecture or if you want to customize the pre-installed packages. The Dockerfiles in `src/` can be modified to achieve this. To use a custom base image, modify `src/Dockerfile.header`. To install specific GPU-related libraries, modify `src/Dockerfile.gpulibs`, and to add specific libraries, append them to `src/Dockerfile.usefulpackages`. +Building a custom Docker image is the recommended option if you have a different GPU architecture or if you want to customize the pre-installed packages. The Dockerfiles in `custom/` can be modified to achieve this. To use a custom base image, modify `custom/header.Dockerfile`. To install specific GPU-related libraries, modify `custom/gpulibs.Dockerfile`, and to add specific libraries, append them to `custom/usefulpackages.Dockerfile`. After making the necessary modifications, regenerate the `Dockerfile` in `/.build`. Once you have confirmed that your GPU is accessible within Docker containers by running `docker run --gpus all nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 nvidia-smi` and seeing the GPU statistics, you can generate, build, and run the Docker image. The following commands will start *GPU-Jupyter* on [localhost:8848](http://localhost:8848) with the default password `gpu-jupyter`. @@ -131,7 +140,7 @@ The following commands will start *GPU-Jupyter* on [localhost:8848](http://local git clone https://github.com/iot-salzburg/gpu-jupyter.git cd gpu-jupyter git branch # Check for extisting branches -git checkout v1.5_cuda-12.0_ubuntu-22.04 # select or create a new version +git checkout v1.6_cuda-12.0_ubuntu-22.04 # select or create a new version # generate the Dockerfile with python and without Julia and R (see options: --help) ./generate-Dockerfile.sh --python-only docker build -t gpu-jupyter .build/ # will take a while @@ -188,7 +197,7 @@ The script has the following parameters: * `-h|--help`: Show a help message. * `-p|--pw|--password`: Set the password for *GPU-Jupyter* by updating - the salted hashed token in `src/jupyter_notebook_config.json`. + the salted hashed token in `custom/jupyter_notebook_config.json`. * `-c|--commit`: specify a commit or `"latest"` for the `docker-stacks`, the default commit is a working one. @@ -198,22 +207,22 @@ As some installations are not needed by everyone, there is the possibility to sk installations to reduce the size of the image. Here the `docker-stack` `scipy-notebook` is used instead of `datascience-notebook` that comes with Julia and R. -Moreover, none of the packages within `src/Dockerfile.usefulpackages` is installed. +Moreover, none of the packages within `custom/usefulpackages.Dockerfile` is installed. * `--python-only|--no-datascience-notebook`: As the name suggests, the `docker-stack` `datascience-notebook` is not installed -on top of the `scipy-notebook`, but the packages within `src/Dockerfile.usefulpackages` are. +on top of the `scipy-notebook`, but the packages within `custom/usefulpackages.Dockerfile` are. * `--no-useful-packages`: On top of the `docker-stack` `datascience-notebook` (Julia and R), -the essential `gpulibs` are installed, but not the packages within `src/Dockerfile.usefulpackages`. +the essential `gpulibs` are installed, but not the packages within `custom/usefulpackages.Dockerfile`. Note that only one of the parameters `--slim`, `--python-only`, and `--no-useful-packages` can be used at the same time: ### Custom Installations -If you need to install custom packages within the container, you can modify the `src/Dockerfile.usefulpackages` file or do it directly within the container. -**Keep in mind that every time a Dockerfile is generated, the file `.build/Dockerfile` is overwritten, so it's best to append custom installations in `src/Dockerfile.usefulpackages` or `generate-Dockerfile.sh`.** +If you need to install custom packages within the container, you can modify the `custom/usefulpackages.Dockerfile` file or do it directly within the container. +**Keep in mind that every time a Dockerfile is generated, the file `.build/Dockerfile` is overwritten, so it's best to append custom installations in `custom/usefulpackages.Dockerfile` or `generate-Dockerfile.sh`.** Some useful packages are suggested in the [Extension docs](https://jupyterlab.readthedocs.io/en/stable/user/extensions.html) and in this blog article from [neptune.ai](https://neptune.ai/blog/jupyterlab-extensions-for-machine-learning). If you notice that an important package is missing in the default stack, please let us know so we can update it. @@ -230,7 +239,7 @@ There are two ways to set a password for GPU-Jupyter: ```bash bash generate-Dockerfile.sh --password [your_password] ``` - This will update automatically the salted hashed token in the `src/jupyter_notebook_config.json` file. Note that the specified password may be visible in your account's bash history. + This will update automatically the salted hashed token in the `custom/jupyter_notebook_config.json` file. Note that the specified password may be visible in your account's bash history. ### Adaptions for using Tensorboard @@ -257,14 +266,14 @@ If the port is exposed, tensorboard can be accessed in the browser on [localhost #### Update CUDA to another version -The GPU-libraries such as PyTorch and Tensorflow in `src/Docker.gpulibs` must support the CUDA version and NVIDIA drivers on the host machine. Check out the compatibility lists for [PyTorch](https://pytorch.org/get-started/locally/) and [Tensorflow](https://www.tensorflow.org/install/source#gpu) or search online for the explicit versions. In this setup, the NVIDIA Driver has version 530.30.02 and CUDA version 11.6.2 is used, which is compatible with Tensorflow 2.10 and PyTorch 1.12. +The GPU-libraries such as PyTorch and Tensorflow in `custom/Docker.gpulibs` must support the CUDA version and NVIDIA drivers on the host machine. Check out the compatibility lists for [PyTorch](https://pytorch.org/get-started/locally/) and [Tensorflow](https://www.tensorflow.org/install/source#gpu) or search online for the explicit versions. In this setup, the NVIDIA Driver has version 530.30.02 and CUDA version 11.6.2 is used, which is compatible with Tensorflow 2.10 and PyTorch 1.12. -The host's CUDA version must be equal to or higher than that used by the container (set within `Dockerfile.header`). +The host's CUDA version must be equal to or higher than that used by the container (set within `custom/header.Dockerfile`). Check the host's version with `nvcc --version` and the version compatibilities for CUDA-dependent packages as [Pytorch](https://pytorch.org/get-started/locally/) respectively [Tensorflow](https://www.tensorflow.org/install/gpu) previously. Then modify, if supported, the CUDA-version (find all tags [here](https://hub.docker.com/r/nvidia/cuda/tags)) -in `src/Dockerfile.header` to, e.g.: +in `custom/header.Dockerfile` to, e.g.: the line: FROM nvidia/cuda:X.Y-base-ubuntu20.04 diff --git a/src/Dockerfile.gpulibs b/custom/gpulibs.Dockerfile similarity index 82% rename from src/Dockerfile.gpulibs rename to custom/gpulibs.Dockerfile index a352c4f..de68319 100644 --- a/src/Dockerfile.gpulibs +++ b/custom/gpulibs.Dockerfile @@ -4,12 +4,6 @@ LABEL maintainer="Christoph Schranz , Mat # https://www.tensorflow.org/install/source#gpu # installation via conda leads to errors in version 4.8.2 # Install CUDA-specific nvidia libraries and update libcudnn8 before that -USER root -RUN apt-get update && \ - apt-get install -y --no-install-recommends --allow-change-held-packages libcudnn8 && \ - apt-get install -y --no-install-recommends libnvinfer-dev libnvinfer-plugin-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* -RUN cd /usr/lib/x86_64-linux-gnu && ln -s libnvinfer_plugin.so.8 libnvinfer_plugin.so.7 && ln -s libnvinfer.so.8 libnvinfer.so.7 USER ${NB_UID} RUN pip install --upgrade pip && \ pip install --no-cache-dir tensorflow==2.15.0 keras==2.15.0 && \ diff --git a/src/Dockerfile.header b/custom/header.Dockerfile similarity index 100% rename from src/Dockerfile.header rename to custom/header.Dockerfile diff --git a/src/Dockerfile.usefulpackages b/custom/usefulpackages.Dockerfile similarity index 100% rename from src/Dockerfile.usefulpackages rename to custom/usefulpackages.Dockerfile diff --git a/generate-Dockerfile.sh b/generate-Dockerfile.sh index 1727373..dbcea37 100755 --- a/generate-Dockerfile.sh +++ b/generate-Dockerfile.sh @@ -5,7 +5,7 @@ cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) export DOCKERFILE=".build/Dockerfile" export STACKS_DIR=".build/docker-stacks" # please test the build of the commit in https://github.com/jupyter/docker-stacks/commits/main in advance -export HEAD_COMMIT="b8d617dc0568d60f6583c42f989da51ec80e9af6" +export HEAD_COMMIT="1494233e27cdc70e3766ea2518e7153ee425fc4f" while [[ "$#" -gt 0 ]]; do case $1 in -p|--pw|--password) PASSWORD="$2" && USE_PASSWORD=1; shift;; @@ -22,10 +22,10 @@ if [[ "$HELP" == 1 ]]; then echo "Help for ./generate-Dockerfile.sh:" echo "Usage: $0 [parameters]" echo " -h|--help: Show this help." - echo " -p|--pw|--password: Set the password (and update in src/jupyter_notebook_config.json)" + echo " -p|--pw|--password: Set the password (and update in custom/jupyter_notebook_config.json)" echo " -c|--commit: Set the head commit of the jupyter/docker-stacks submodule (https://github.com/jupyter/docker-stacks/commits/main). default: $HEAD_COMMIT." echo " --python-only|--no-datascience-notebook: Use not the datascience-notebook from jupyter/docker-stacks, don't install Julia and R." - echo " --no-useful-packages: Don't install the useful packages, specified in src/Dockerfile.usefulpackages" + echo " --no-useful-packages: Don't install the useful packages, specified in custom/usefulpackages.Dockerfile" echo " --slim: no useful packages and no datascience notebook." exit 21 fi @@ -52,39 +52,58 @@ else fi # Write the contents into the DOCKERFILE and start with the header -echo "# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'src/' +echo "# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'custom/' # **Please do not change this file directly!** -# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'src/Dockerfile.usefulpackages'. +# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'custom/usefulpackages.Dockerfile'. # More information can be found in the README under configuration. " > $DOCKERFILE -cat src/Dockerfile.header >> $DOCKERFILE +cat custom/header.Dockerfile >> $DOCKERFILE echo " ############################################################################ #################### Dependency: jupyter/docker-stacks-foundation ########## ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE +if [ -f "$STACKS_DIR/images/docker-stacks-foundation/Dockerfile" ]; then + cat $STACKS_DIR/images/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/images/docker-stacks-foundation/initial-condarc .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/fix-permissions .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/start.sh .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/run-hooks.sh .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/10activate-conda-env.sh .build/ +else + cat $STACKS_DIR/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/docker-stacks-foundation/initial-condarc .build/ + cp $STACKS_DIR/docker-stacks-foundation/fix-permissions .build/ + cp $STACKS_DIR/docker-stacks-foundation/start.sh .build/ +fi echo " ############################################################################ #################### Dependency: jupyter/base-notebook ##################### ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE - -# copy files that are used during the build: -cp $STACKS_DIR/docker-stacks-foundation/initial-condarc .build/ -cp $STACKS_DIR/docker-stacks-foundation/fix-permissions .build/ -cp $STACKS_DIR/docker-stacks-foundation/start.sh .build/ -cp $STACKS_DIR/base-notebook/jupyter_server_config.py .build/ -cp $STACKS_DIR/base-notebook/start-notebook.sh .build/ -cp $STACKS_DIR/base-notebook/start-singleuser.sh .build/ -cp $STACKS_DIR/base-notebook/docker_healthcheck.py .build/ -cp -r $STACKS_DIR/minimal-notebook/setup-scripts .build/ -cp $STACKS_DIR/minimal-notebook/Rprofile.site .build/ +if [ -f "$STACKS_DIR/images/base-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/images/base-notebook/jupyter_server_config.py .build/ + cp $STACKS_DIR/images/base-notebook/start-notebook.sh .build/ + cp $STACKS_DIR/images/base-notebook/start-notebook.py .build/ + cp $STACKS_DIR/images/base-notebook/start-singleuser.sh .build/ + cp $STACKS_DIR/images/base-notebook/start-singleuser.py .build/ + cp $STACKS_DIR/images/base-notebook/docker_healthcheck.py .build/ +else + cat $STACKS_DIR/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/base-notebook/jupyter_server_config.py .build/ + cp $STACKS_DIR/base-notebook/start-notebook.sh .build/ + cp $STACKS_DIR/base-notebook/start-singleuser.sh .build/ + cp $STACKS_DIR/base-notebook/docker_healthcheck.py .build/ +fi chmod 755 .build/* echo " @@ -92,14 +111,28 @@ echo " ################# Dependency: jupyter/minimal-notebook ##################### ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +if [ -f "$STACKS_DIR/images/minimal-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + # copy files that are used during the build + cp -r $STACKS_DIR/images/minimal-notebook/setup-scripts .build/ + cp $STACKS_DIR/images/minimal-notebook/Rprofile.site .build/ +else + cat $STACKS_DIR/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + # copy files that are used during the build + cp -r $STACKS_DIR/minimal-notebook/setup-scripts .build/ + cp $STACKS_DIR/minimal-notebook/Rprofile.site .build/ +fi echo " ############################################################################ ################# Dependency: jupyter/scipy-notebook ####################### ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +if [ -f "$STACKS_DIR/images/scipy-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +else + cat $STACKS_DIR/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +fi # install Julia and R if not excluded or spare mode is used if [[ "$no_datascience_notebook" != 1 ]]; then @@ -108,7 +141,11 @@ if [[ "$no_datascience_notebook" != 1 ]]; then ################ Dependency: jupyter/datascience-notebook ################## ############################################################################ " >> $DOCKERFILE - cat $STACKS_DIR/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + if [ -f "$STACKS_DIR/images/datascience-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + else + cat $STACKS_DIR/images/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + fi else echo "Set 'no-datascience-notebook' = 'python-only', not installing the datascience-notebook with Julia and R." fi @@ -120,7 +157,7 @@ echo " ########################## Dependency: gpulibs ############################# ############################################################################ " >> $DOCKERFILE -cat src/Dockerfile.gpulibs >> $DOCKERFILE +cat custom/gpulibs.Dockerfile >> $DOCKERFILE # install useful packages if not excluded or spare mode is used if [[ "$no_useful_packages" != 1 ]]; then @@ -129,9 +166,9 @@ if [[ "$no_useful_packages" != 1 ]]; then ############################ Useful packages ############################### ############################################################################ " >> $DOCKERFILE - cat src/Dockerfile.usefulpackages >> $DOCKERFILE + cat custom/usefulpackages.Dockerfile >> $DOCKERFILE else - echo "Set 'no-useful-packages', not installing stuff within src/Dockerfile.usefulpackages." + echo "Set 'no-useful-packages', not installing stuff within custom/usefulpackages.Dockerfile." fi # Copy the demo notebooks and change permissions