build: extend dockerfiles to support linux arm64 builds

Generated a requirements file for arm64-compatible tensorflow-cpu image.
usnistgov · Sep 26, 2023 · 88ddf58 · 88ddf58
1 parent 3f49395
commit 88ddf58
Show file tree

Hide file tree

Showing 16 changed files with 627 additions and 2,851 deletions.
diff --git a/.github/workflows/pip-compile.yml b/.github/workflows/pip-compile.yml
@@ -177,7 +177,7 @@ jobs:
           - "py39-linux-aarch64-mlflow-tracking-requirements"
           - "py39-linux-aarch64-restapi-requirements"
           - "py39-linux-aarch64-pytorch-cpu-requirements"
-          # - "py39-linux-aarch64-tensorflow2-cpu-requirements"
+          - "py39-linux-aarch64-tensorflow2-cpu-requirements"
 
     steps:
       - uses: actions/checkout@v4

diff --git a/Makefile b/Makefile
@@ -46,17 +46,17 @@ DETECTED_ARCH := x86_64
 else ifeq ($(ARCH),amd64)
 DETECTED_ARCH := x86_64
 else ifeq ($(ARCH),aarch64)
-DETECTED_ARCH := aarch64
+DETECTED_ARCH := arm64
 else ifeq ($(ARCH),arm64)
-DETECTED_ARCH := aarch64
+DETECTED_ARCH := arm64
 endif
 
 VENV_EXTRA ?=
 
 ifeq ($(DETECTED_OS),Darwin)
 CORES = $(shell sysctl -n hw.physicalcpu_max)
-PIPTOOLS_SYNC := CFLAGS="-stdlib=libc++" pip-sync
-VENV_REQUIREMENTS = requirements/macos-$(if $(filter aarch64, $(DETECTED_ARCH)),arm64,x86_64)-py$(PYTHON_VERSION_MAJOR).$(PYTHON_VERSION_MINOR)-requirements-dev$(VENV_EXTRA).txt
+PIPTOOLS_SYNC := CFLAGS="-stdlib=libc++ -std=c99" pip-sync
+VENV_REQUIREMENTS = requirements/macos-$(DETECTED_ARCH)-py$(PYTHON_VERSION_MAJOR).$(PYTHON_VERSION_MINOR)-requirements-dev$(VENV_EXTRA).txt
 else ifeq ($(DETECTED_OS),Linux)
 CORES = $(shell lscpu -p | egrep -v '^\#' | sort -u -t, -k 2,4 | wc -l)
 PIPTOOLS_SYNC := pip-sync
@@ -181,7 +181,7 @@ DOCS_WEB_COMPILE_FILES := $(wildcard $(DOCS_SCSS_DIR)/*.scss)
 
 PIP :=
 ifeq ($(DETECTED_OS),Darwin)
-PIP += CFLAGS="-stdlib=libc++" $(PY) -m pip
+PIP += CFLAGS="-stdlib=libc++ -std=c99" $(PY) -m pip
 else
 PIP += $(PY) -m pip
 endif

diff --git a/README.md b/README.md
@@ -35,9 +35,9 @@ Ensure that you have Python 3.9 installed and that it is available in your PATH,
 | linux-x86_64-py3.9-requirements-dev.txt | Linux | x86-64 | ❌ | ❌ |
 | linux-x86_64-py3.9-requirements-dev-tensorflow.txt | Linux | x86-64 | ✅ | ❌ |
 | linux-x86_64-py3.9-requirements-dev-pytorch.txt | Linux | x86-64 | ❌ | ✅ |
-| linux-aarch64-py3.9-requirements-dev.txt | Linux | arm64 | ❌ | ❌ |
-| linux-aarch64-py3.9-requirements-dev-tensorflow.txt | Linux | arm64 | ❌ | ❌ |
-| linux-aarch64-py3.9-requirements-dev-pytorch.txt | Linux | arm64 | ❌ | ✅ |
+| linux-arm64-py3.9-requirements-dev.txt | Linux | arm64 | ❌ | ❌ |
+| linux-arm64-py3.9-requirements-dev-tensorflow.txt | Linux | arm64 | ❌ | ❌ |
+| linux-arm64-py3.9-requirements-dev-pytorch.txt | Linux | arm64 | ❌ | ✅ |
 | macos-x86_64-py3.9-requirements-dev.txt | MacOS | x86-64 | ❌ | ❌ |
 | macos-x86_64-py3.9-requirements-dev-tensorflow.txt | MacOS | x86-64 | ✅ | ❌ |
 | macos-x86_64-py3.9-requirements-dev-pytorch.txt | MacOS | x86-64 | ❌ | ✅ |

diff --git a/docker/Dockerfile.mlflow-tracking b/docker/Dockerfile.mlflow-tracking
@@ -23,9 +23,11 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-mlflow-tracking-requirements.txt /files/mlflow-tracking-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-mlflow-tracking-requirements.txt /files/mlflow-tracking-requirements.txt
 
 ######################################################################################################
 # Base images
@@ -148,13 +150,16 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${TARGETARCH/arm/aarch}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/docker/Dockerfile.pytorch-cpu b/docker/Dockerfile.pytorch-cpu
@@ -23,9 +23,11 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-pytorch-cpu-requirements.txt /files/pytorch-cpu-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-pytorch-cpu-requirements.txt /files/pytorch-cpu-requirements.txt
 
 ######################################################################################################
 # Base images
@@ -160,13 +162,16 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${TARGETARCH/arm/aarch}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/docker/Dockerfile.restapi b/docker/Dockerfile.restapi
@@ -23,11 +23,13 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root src/migrations /files/migrations
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
 COPY --chown=root:root --chmod=0644 docker/configs/gunicorn.restapi.conf.py /files/gunicorn.restapi.conf.py
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-restapi-requirements.txt /files/restapi-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-restapi-requirements.txt /files/restapi-requirements.txt
 COPY --chown=root:root --chmod=0755 wsgi.py /files/wsgi.py
 
 RUN chmod 0755 /files/migrations && \
@@ -164,13 +166,16 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${TARGETARCH/arm/aarch}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/docker/Dockerfile.tensorflow2-cpu b/docker/Dockerfile.tensorflow2-cpu
@@ -23,9 +23,11 @@
 
 FROM ubuntu:focal AS copy-files
 
+ARG TARGETARCH
+
 COPY --chown=root:root --chmod=0644 docker/configs/aws-config /files/aws-config
 COPY --chown=root:root --chmod=0644 docker/configs/build.pip.conf /files/build.pip.conf
-COPY --chown=root:root --chmod=0644 docker/requirements/linux-x86_64-py3.9-tensorflow2-cpu-requirements.txt /files/tensorflow2-cpu-requirements.txt
+COPY --chown=root:root --chmod=0644 docker/requirements/linux-${TARGETARCH}-py3.9-tensorflow2-cpu-requirements.txt /files/tensorflow2-cpu-requirements.txt
 
 ######################################################################################################
 # Base images
@@ -160,13 +162,16 @@ RUN mkdir -p /shellscripts && \
 
 FROM certs-base AS install-awscli
 
-ARG AWSCLI_DOWNLOAD_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+ARG TARGETARCH
+ARG AWSCLI_DOWNLOAD_BASE_URL=https://awscli.amazonaws.com
 ARG AWSCLI_INSTALL_DIR=/opt/aws-cli
 ARG AWSCLI_BIN_DIR=/usr/local/bin
 
+SHELL ["/bin/bash", "-c"]
+
 RUN cd /tmp && \
-    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_URL} && \
-    unzip awscliv2.zip && \
+    wget -qO /tmp/awscliv2.zip ${AWSCLI_DOWNLOAD_BASE_URL}/awscli-exe-linux-${TARGETARCH/arm/aarch}.zip && \
+    unzip /tmp/awscliv2.zip && \
     rm /tmp/awscliv2.zip && \
     ./aws/install --install-dir ${AWSCLI_INSTALL_DIR} --bin-dir ${AWSCLI_BIN_DIR} && \
     rm -rf /tmp/aws

diff --git a/...64-py3.9-mlflow-tracking-requirements.txt → ...64-py3.9-mlflow-tracking-requirements.txt b/...64-py3.9-mlflow-tracking-requirements.txt → ...64-py3.9-mlflow-tracking-requirements.txt
diff --git a/...arch64-py3.9-pytorch-cpu-requirements.txt → ...-arm64-py3.9-pytorch-cpu-requirements.txt b/...arch64-py3.9-pytorch-cpu-requirements.txt → ...-arm64-py3.9-pytorch-cpu-requirements.txt
diff --git a/...ux-aarch64-py3.9-restapi-requirements.txt → ...inux-arm64-py3.9-restapi-requirements.txt b/...ux-aarch64-py3.9-restapi-requirements.txt → ...inux-arm64-py3.9-restapi-requirements.txt