build: extend dockerfiles to support linux arm64 builds

Generated a requirements file for arm64-compatible tensorflow-cpu image. Normalized the architecture names to amd64 and arm64. Closes #206 Co-authored-by: James K. Glasbrenner <[email protected]>
usnistgov · Sep 26, 2023 · 5b6087e · 5b6087e
1 parent 3f49395
commit 5b6087e
Show file tree

Hide file tree

Showing 34 changed files with 678 additions and 88 deletions.
diff --git a/.github/workflows/pip-compile.yml b/.github/workflows/pip-compile.yml
@@ -177,7 +177,7 @@ jobs:
           - "py39-linux-aarch64-mlflow-tracking-requirements"
           - "py39-linux-aarch64-restapi-requirements"
           - "py39-linux-aarch64-pytorch-cpu-requirements"
-          # - "py39-linux-aarch64-tensorflow2-cpu-requirements"
+          - "py39-linux-aarch64-tensorflow2-cpu-requirements"
 
     steps:
       - uses: actions/checkout@v4

diff --git a/Makefile b/Makefile
@@ -42,21 +42,21 @@ PYTHON_VERSION_MINOR := $(word 2,$(subst ., ,$(PYTHON_VERSION)))
 ARCH := $(strip $(shell /usr/bin/env $(PY) -c 'import platform; print(platform.machine().lower())'))
 
 ifeq ($(ARCH),x86_64)
-DETECTED_ARCH := x86_64
+DETECTED_ARCH := amd64
 else ifeq ($(ARCH),amd64)
-DETECTED_ARCH := x86_64
+DETECTED_ARCH := amd64
 else ifeq ($(ARCH),aarch64)
-DETECTED_ARCH := aarch64
+DETECTED_ARCH := arm64
 else ifeq ($(ARCH),arm64)
-DETECTED_ARCH := aarch64
+DETECTED_ARCH := arm64
 endif
 
 VENV_EXTRA ?=
 
 ifeq ($(DETECTED_OS),Darwin)
 CORES = $(shell sysctl -n hw.physicalcpu_max)
-PIPTOOLS_SYNC := CFLAGS="-stdlib=libc++" pip-sync
-VENV_REQUIREMENTS = requirements/macos-$(if $(filter aarch64, $(DETECTED_ARCH)),arm64,x86_64)-py$(PYTHON_VERSION_MAJOR).$(PYTHON_VERSION_MINOR)-requirements-dev$(VENV_EXTRA).txt
+PIPTOOLS_SYNC := CFLAGS="-stdlib=libc++ -std=c99" pip-sync
+VENV_REQUIREMENTS = requirements/macos-$(DETECTED_ARCH)-py$(PYTHON_VERSION_MAJOR).$(PYTHON_VERSION_MINOR)-requirements-dev$(VENV_EXTRA).txt
 else ifeq ($(DETECTED_OS),Linux)
 CORES = $(shell lscpu -p | egrep -v '^\#' | sort -u -t, -k 2,4 | wc -l)
 PIPTOOLS_SYNC := pip-sync
@@ -181,7 +181,7 @@ DOCS_WEB_COMPILE_FILES := $(wildcard $(DOCS_SCSS_DIR)/*.scss)
 
 PIP :=
 ifeq ($(DETECTED_OS),Darwin)
-PIP += CFLAGS="-stdlib=libc++" $(PY) -m pip
+PIP += CFLAGS="-stdlib=libc++ -std=c99" $(PY) -m pip
 else
 PIP += $(PY) -m pip
 endif

diff --git a/README.md b/README.md
@@ -32,21 +32,21 @@ Ensure that you have Python 3.9 installed and that it is available in your PATH,
 
 | Filename | OS | Architecture | Tensorflow | PyTorch |
 | :--- | :---: | :---: | :--- | :--- |
-| linux-x86_64-py3.9-requirements-dev.txt | Linux | x86-64 | ❌ | ❌ |
-| linux-x86_64-py3.9-requirements-dev-tensorflow.txt | Linux | x86-64 | ✅ | ❌ |
-| linux-x86_64-py3.9-requirements-dev-pytorch.txt | Linux | x86-64 | ❌ | ✅ |
-| linux-aarch64-py3.9-requirements-dev.txt | Linux | arm64 | ❌ | ❌ |
-| linux-aarch64-py3.9-requirements-dev-tensorflow.txt | Linux | arm64 | ❌ | ❌ |
-| linux-aarch64-py3.9-requirements-dev-pytorch.txt | Linux | arm64 | ❌ | ✅ |
-| macos-x86_64-py3.9-requirements-dev.txt | MacOS | x86-64 | ❌ | ❌ |
-| macos-x86_64-py3.9-requirements-dev-tensorflow.txt | MacOS | x86-64 | ✅ | ❌ |
-| macos-x86_64-py3.9-requirements-dev-pytorch.txt | MacOS | x86-64 | ❌ | ✅ |
+| linux-amd64-py3.9-requirements-dev.txt | Linux | x86-64 | ❌ | ❌ |
+| linux-amd64-py3.9-requirements-dev-tensorflow.txt | Linux | x86-64 | ✅ | ❌ |
+| linux-amd64-py3.9-requirements-dev-pytorch.txt | Linux | x86-64 | ❌ | ✅ |
+| linux-arm64-py3.9-requirements-dev.txt | Linux | arm64 | ❌ | ❌ |
+| linux-arm64-py3.9-requirements-dev-tensorflow.txt | Linux | arm64 | ❌ | ❌ |
+| linux-arm64-py3.9-requirements-dev-pytorch.txt | Linux | arm64 | ❌ | ✅ |
+| macos-amd64-py3.9-requirements-dev.txt | MacOS | x86-64 | ❌ | ❌ |
+| macos-amd64-py3.9-requirements-dev-tensorflow.txt | MacOS | x86-64 | ✅ | ❌ |
+| macos-amd64-py3.9-requirements-dev-pytorch.txt | MacOS | x86-64 | ❌ | ✅ |
 | macos-arm64-py3.9-requirements-dev.txt | MacOS | arm64 | ❌ | ❌ |
 | macos-arm64-py3.9-requirements-dev-tensorflow.txt | MacOS | arm64 | ✅ | ❌ |
 | macos-arm64-py3.9-requirements-dev-pytorch.txt | MacOS | arm64 | ❌ | ✅ |
-| win-x86_64-py3.9-requirements-dev.txt | Windows | x86-64 | ❌ | ❌ |
-| win-x86_64-py3.9-requirements-dev-tensorflow.txt | Windows | x86-64 | ✅ | ❌ |
-| win-x86_64-py3.9-requirements-dev-pytorch.txt | Windows | x86-64 | ❌ | ✅ |
+| win-amd64-py3.9-requirements-dev.txt | Windows | x86-64 | ❌ | ❌ |
+| win-amd64-py3.9-requirements-dev-tensorflow.txt | Windows | x86-64 | ✅ | ❌ |
+| win-amd64-py3.9-requirements-dev-pytorch.txt | Windows | x86-64 | ❌ | ✅ |
 
 Next, use the `venv` module to create a new virtual environment:
 
@@ -77,15 +77,15 @@ Finally, use `pip-sync` to install the dependencies in your chosen requirements
 On MacOS/Linux:
 
 ```sh
-# Replace "linux-x86_64-py3.9-requirements-dev.txt" with your chosen file
-pip-sync requirements/linux-x86_64-py3.9-requirements-dev.txt
+# Replace "linux-amd64-py3.9-requirements-dev.txt" with your chosen file
+pip-sync requirements/linux-amd64-py3.9-requirements-dev.txt
 ```
 
 On Windows:
 
 ```powershell
-# Replace "win-x86_64-py3.9-requirements-dev.txt" with your chosen file
-pip-sync requirements\win-x86_64-py3.9-requirements-dev.txt
+# Replace "win-amd64-py3.9-requirements-dev.txt" with your chosen file
+pip-sync requirements\win-amd64-py3.9-requirements-dev.txt
 ```
 
 If the requirements file you used is updated, or if you want to switch to another requirements file (you need access to the Tensorflow library, for example), just run `pip-sync` again using the appropriate filename.

diff --git a/container-vars.mk b/container-vars.mk
@@ -24,7 +24,7 @@
 CONTAINER_MLFLOW_TRACKING_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-x86_64-py3.9-mlflow-tracking-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-mlflow-tracking-requirements.txt\
     docker/shellscripts/entrypoint-mlflow-tracking.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -47,7 +47,7 @@ CONTAINER_NGINX_INCLUDE_FILES =\
 CONTAINER_PYTORCH_CPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-x86_64-py3.9-pytorch-cpu-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-pytorch-cpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -65,7 +65,7 @@ CONTAINER_PYTORCH_CPU_INCLUDE_FILES =\
 CONTAINER_PYTORCH_GPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-x86_64-py3.9-pytorch-gpu-requirements.txt\
+    docker/requirements/linux-amd64-py3.9-pytorch-gpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -84,7 +84,7 @@ CONTAINER_RESTAPI_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
     docker/configs/gunicorn.restapi.conf.py\
-    docker/requirements/linux-x86_64-py3.9-restapi-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-restapi-requirements.txt\
     docker/shellscripts/entrypoint-restapi.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -103,7 +103,7 @@ CONTAINER_RESTAPI_INCLUDE_FILES =\
 CONTAINER_TENSORFLOW2_CPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-x86_64-py3.9-tensorflow2-cpu-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-tensorflow2-cpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -121,7 +121,7 @@ CONTAINER_TENSORFLOW2_CPU_INCLUDE_FILES =\
 CONTAINER_TENSORFLOW2_GPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-x86_64-py3.9-tensorflow2-gpu-requirements.txt\
+    docker/requirements/linux-amd64-py3.9-tensorflow2-gpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\

diff --git a/docker/Dockerfile.mlflow-tracking b/docker/Dockerfile.mlflow-tracking
@@ -23,9 +23,11 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-mlflow-tracking-requirements.txt /files/mlflow-tracking-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-mlflow-tracking-requirements.txt /files/mlflow-tracking-requirements.txt
 
 ######################################################################################################
 # Base images
@@ -148,13 +150,17 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    AWSCLIARCH=${TARGETARCH/arm/aarch} && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${AWSCLIARCH/amd/x86_}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/docker/Dockerfile.pytorch-cpu b/docker/Dockerfile.pytorch-cpu
@@ -23,9 +23,11 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-pytorch-cpu-requirements.txt /files/pytorch-cpu-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-pytorch-cpu-requirements.txt /files/pytorch-cpu-requirements.txt
 
 ######################################################################################################
 # Base images
@@ -160,13 +162,17 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    AWSCLIARCH=${TARGETARCH/arm/aarch} && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${AWSCLIARCH/amd/x86_}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/docker/Dockerfile.pytorch-gpu b/docker/Dockerfile.pytorch-gpu
@@ -25,7 +25,7 @@ FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04 AS copy-files
 
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-pytorch-gpu-requirements.txt /files/pytorch-gpu-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-amd64-py3.9-pytorch-gpu-requirements.txt /files/pytorch-gpu-requirements.txt
 
 ######################################################################################################
 # Base images

diff --git a/docker/Dockerfile.restapi b/docker/Dockerfile.restapi
@@ -23,11 +23,13 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root src/migrations /files/migrations
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
 COPY --chown=root:root --chmod=0644 docker/configs/gunicorn.restapi.conf.py /files/gunicorn.restapi.conf.py
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-restapi-requirements.txt /files/restapi-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-restapi-requirements.txt /files/restapi-requirements.txt
 COPY --chown=root:root --chmod=0755 wsgi.py /files/wsgi.py
 
 RUN chmod 0755 /files/migrations && \
@@ -164,13 +166,17 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    AWSCLIARCH=${TARGETARCH/arm/aarch} && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${AWSCLIARCH/amd/x86_}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/docker/Dockerfile.tensorflow2-cpu b/docker/Dockerfile.tensorflow2-cpu
@@ -23,9 +23,11 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-tensorflow2-cpu-requirements.txt /files/tensorflow2-cpu-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-tensorflow2-cpu-requirements.txt /files/tensorflow2-cpu-requirements.txt
 
 ######################################################################################################
 # Base images
@@ -160,13 +162,17 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    AWSCLIARCH=${TARGETARCH/arm/aarch} && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${AWSCLIARCH/amd/x86_}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/docker/Dockerfile.tensorflow2-gpu b/docker/Dockerfile.tensorflow2-gpu
@@ -25,7 +25,7 @@ FROM nvidia/cuda:12.0.0-cudnn8-runtime-ubuntu20.04 AS copy-files
 
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-tensorflow2-gpu-requirements.txt /files/tensorflow2-gpu-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-amd64-py3.9-tensorflow2-gpu-requirements.txt /files/tensorflow2-gpu-requirements.txt
 
 ######################################################################################################
 # Base images

diff --git a/...64-py3.9-mlflow-tracking-requirements.txt → ...64-py3.9-mlflow-tracking-requirements.txt b/...64-py3.9-mlflow-tracking-requirements.txt → ...64-py3.9-mlflow-tracking-requirements.txt
diff --git a/...x86_64-py3.9-pytorch-cpu-requirements.txt → ...-amd64-py3.9-pytorch-cpu-requirements.txt b/...x86_64-py3.9-pytorch-cpu-requirements.txt → ...-amd64-py3.9-pytorch-cpu-requirements.txt
diff --git a/...x86_64-py3.9-pytorch-gpu-requirements.txt → ...-amd64-py3.9-pytorch-gpu-requirements.txt b/...x86_64-py3.9-pytorch-gpu-requirements.txt → ...-amd64-py3.9-pytorch-gpu-requirements.txt
diff --git a/...nux-x86_64-py3.9-restapi-requirements.txt → ...inux-amd64-py3.9-restapi-requirements.txt b/...nux-x86_64-py3.9-restapi-requirements.txt → ...inux-amd64-py3.9-restapi-requirements.txt
diff --git a/...64-py3.9-tensorflow2-cpu-requirements.txt → ...64-py3.9-tensorflow2-cpu-requirements.txt b/...64-py3.9-tensorflow2-cpu-requirements.txt → ...64-py3.9-tensorflow2-cpu-requirements.txt
diff --git a/...64-py3.9-tensorflow2-gpu-requirements.txt → ...64-py3.9-tensorflow2-gpu-requirements.txt b/...64-py3.9-tensorflow2-gpu-requirements.txt → ...64-py3.9-tensorflow2-gpu-requirements.txt
diff --git a/...64-py3.9-mlflow-tracking-requirements.txt → ...64-py3.9-mlflow-tracking-requirements.txt b/...64-py3.9-mlflow-tracking-requirements.txt → ...64-py3.9-mlflow-tracking-requirements.txt
diff --git a/...arch64-py3.9-pytorch-cpu-requirements.txt → ...-arm64-py3.9-pytorch-cpu-requirements.txt b/...arch64-py3.9-pytorch-cpu-requirements.txt → ...-arm64-py3.9-pytorch-cpu-requirements.txt
diff --git a/...ux-aarch64-py3.9-restapi-requirements.txt → ...inux-arm64-py3.9-restapi-requirements.txt b/...ux-aarch64-py3.9-restapi-requirements.txt → ...inux-arm64-py3.9-restapi-requirements.txt