Merge branch 'main' into njit_parallelization

.github/workflows/continuous-deployment.yml

@@ -0,0 +1,89 @@
+name: Continuous Deployment
+
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+  delete:
+
+jobs:
+  build-and-push-image:
+    runs-on: self-hosted
+    if: ${{ github.event_name == 'push' }}
+    steps:
+      - name: Create Release
+        id: set-release
+        if: ${{ github.ref_name == 'main' }}
+        # Only create the automated release PR if pushing to main (won't be created if this is already the automated release PR)
+        uses: google-github-actions/release-please-action@v3
+        with:
+          release-type: python
+          package-name: ${{ github.event.repository.name }}
+      - name: Checkout code
+        id: set-code
+        if: ${{ (github.ref_name == 'main' && steps.set-release.outputs.release_created) || github.ref_name != 'main'}}
+        # Only perform next steps if now merging the automated release PR to main, or if not pushing to main
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Setup SSH Agent
+        if: ${{ (github.ref_name == 'main' && steps.set-release.outputs.release_created) || github.ref_name != 'main'}}
+        uses: webfactory/[email protected]
+        id: set-ssh
+        with:
+          ssh-private-key: ${{ secrets.SPS_SSH_ID }}
+      - name: Setup Docker Buildx
+        id: set-docker-buildx
+        if: ${{ (github.ref_name == 'main' && steps.set-release.outputs.release_created) || github.ref_name != 'main'}}
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+      - name: Perform DockerHub Login
+        id: set-dockerhub-login
+        if: ${{ (github.ref_name == 'main' && steps.set-release.outputs.release_created) || github.ref_name != 'main'}}
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      # Push two images, one with latest as tag (overwriting), and one with its version as tag (for future reference)
+      - name: Build Docker Image and Push to DockerHub
+        id: set-build-and-push-latest
+        if: ${{ (github.ref_name == 'main' && steps.set-release.outputs.release_created)}}
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: Dockerfile
+          target: runtime
+          tags: |
+            chimefrb/${{ github.event.repository.name }}:latest
+            chimefrb/${{ github.event.repository.name }}:${{ steps.set-release.outputs.tag_name }}
+          ssh: "github_ssh_id=${{ steps.set-ssh.outputs.SSH_AUTH_SOCK}}"
+          push: true
+      # Push the image with just its branch as tag
+      - name: Build Docker Image and Push to DockerHub
+        id: set-build-and-push-branch
+        if: ${{ github.ref_name != 'main'}}
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: Dockerfile
+          target: runtime
+          tags: |
+            chimefrb/${{ github.event.repository.name }}:${{ github.ref_name }}
+          ssh: "github_ssh_id=${{ steps.set-ssh.outputs.SSH_AUTH_SOCK}}"
+          push: true
+  delete-image:
+    runs-on: ubuntu-latest
+    # If a pull request is merged, or a branch is deleted, delete the associated DockerHub Image Tag
+    if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.merged && github.actor != 'github-actions' && github.base_ref == 'main') || github.event.ref_type == 'branch'}}
+    steps:
+      - name: Delete Docker Image Tag of Branch from DockerHub
+        run: |
+          TAG='${{ github.head_ref || github.event.ref}}'
+          HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"${{ secrets.DOCKERHUB_USERNAME }}\", \"password\": \"${{ secrets.DOCKERHUB_PASSWORD }}\"}" https://hub.docker.com/v2/users/login/ | jq -r .token)
+          curl -i -X DELETE \ -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/${{ secrets.DOCKERHUB_USERNAME }}/repositories/${{ github.event.repository.name }}/tags/$TAG/

.github/workflows/continuous-integration.yml

@@ -0,0 +1,91 @@
+name: Continuous Integration
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - 'docs/**'
+
+env:
+  ENV PIP_NO_CACHE_DIR: true
+  ENV OPENMP_ENABLED: 0
+  PYTHONUNBUFFERED: 1
+  PYTHONDONTWRITEBYTECODE: 1
+  PIP_NO_CACHE_DIR: true
+  PIP_DISABLE_PIP_VERSION_CHECK: on
+  PIP_DEFAULT_TIMEOUT: 100
+  POETRY_VERSION: 1.6.1
+  POETRY_NO_INTERACTION: 1
+  DEBIAN_FRONTEND: noninteractive
+  PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
+
+jobs:
+  pre-commit-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup code repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Setup Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Load cached pre-commit repos
+        id: cached-pre-commit-repos
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pre-commit
+          key: precommit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
+      - name: Get all changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v36
+      - name: Setup ssh agent
+        uses: webfactory/[email protected]
+        with:
+            ssh-private-key: ${{ secrets.SPS_SSH_ID }}
+      - name: Perform pre-commit checks
+        run: |
+            pip install pre-commit
+            pre-commit run --files ${{ steps.changed-files.outputs.all_changed_files }}
+  github-tests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        python-version: [ "3.8" ]
+    steps:
+      - name: Setup code repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+            python-version: ${{ matrix.python-version }}
+      - name: Install poetry
+        run: |
+          pip install poetry==$POETRY_VERSION
+      - name: Setup ssh agent
+        uses: webfactory/[email protected]
+        with:
+            ssh-private-key: ${{ secrets.SPS_SSH_ID }}
+      - name: Install linux-dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -yqq --no-install-recommends curl ssh git build-essential
+      - name: Install requirements
+        run: |
+          poetry install
+      - name: Run tests
+        run: |
+          echo "Running tests!"
+          poetry run pytest
+          echo "Finished running tests!"

.pre-commit-config.yaml

@@ -0,0 +1,132 @@
+exclude: ^(docs/.*|tests/.*|data/.*|.venv/.*)$
+repos:
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.3.1
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py38-plus
+
+  - repo: https://github.com/MarcoGorelli/absolufy-imports
+    rev: v0.3.1
+    hooks:
+      - id: absolufy-imports
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        args:
+          - --profile=black
+          - --verbose
+
+  - repo: https://github.com/pycqa/autoflake
+    rev: v2.2.0
+    hooks:
+      - id: autoflake
+        args:
+          - --in-place # Overwrite contents
+          - --remove-all-unused-imports
+
+  - repo: https://github.com/pycqa/docformatter
+    rev: v1.7.5
+    hooks:
+      - id: docformatter
+        args:
+          # Add a "--style=numpy" or "--style=google" once support is added
+          - --in-place
+          - --black # Wraps descriptions and summaries if over 88 length
+          - --pre-summary-newline
+          - --close-quotes-on-newline
+          # Avoid adding "--make-summary-multiline" to enforce no single-line docstrings
+          # as this conflicts with Black and causes perpetual formatting by both of them
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        args:
+          - --preview # Allows Black to fix "line too long" issues in strings
+          - --verbose
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-ast
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-yaml
+      - id: debug-statements
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          # Add to these types as needed to allow for type checking, based on various module imports
+          - types-attrs
+          - types-pytz
+          - types-PyYAML
+          - types-requests
+          - types-setuptools
+          - types-toml
+        args:
+          # Add --strict after type-hinting addition first pass merged, to enforce everything is typed
+          - --ignore-missing-imports # mypy uses a local venv, so ignore the inevitable missing sps package import errors
+          - --follow-imports=skip # If you follow imports, mypy will check more than just the diff
+          - --no-strict-optional # Allows None to be a valid for every type
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-bugbear==23.7.10
+        args:
+          # - --max-complexity=15 Too dificult to fix per pull-request, needs a separate code review
+          - --max-line-length=88 # To be equivalent to Black's format
+          - --extend-select=B950
+          - --extend-ignore=E203,E402,E731,E501,B0
+          # E501: Bugbear’s B950 will replace this, allowing Black’s 10% rule
+          # E402: There are many instances where we need to import SPS packages AFTER setting variables
+          # E203: Black formats in such a way that this error will always be triggered
+          # E731: Allow assigning of lambda expressions, they're useful
+          # B0: Ignore all other Bugbear errors (not needed)
+
+  - repo: https://github.com/pycqa/pydocstyle
+    rev: 6.3.0
+    hooks:
+      - id: pydocstyle
+        args:
+          - --convention=pep257 # Change to "numpy" or "google" once support is added to pycqa/docformatter
+          - --add-ignore=D400,D401
+          # - --add-ignre=D100,D101,D102,D103,D104,D105,D106,D107
+          # D1XX: the above argument, if uncommented, disables missing docstring errors
+          # D400/D401: semantic errors
+
+  - repo: https://github.com/pycqa/bandit
+    rev: 1.7.4
+    hooks:
+      - id: bandit
+        args:
+          - --skip=B101 # To allow usage of asserts
+
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.6.1
+    hooks:
+      - id: poetry-check
+      - id: poetry-lock
+
+  - repo: https://github.com/commitizen-tools/commitizen
+    rev: v3.4.0
+    hooks:
+      - id: commitizen
+        stages:
+          - commit-msg # Enforce commitizen on commit message attempts

Dockerfile

@@ -0,0 +1,51 @@
+FROM python:3.11-slim as runtime
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VERSION=1.6.1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    DEBIAN_FRONTEND=noninteractive
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -yqq --no-install-recommends \
+    curl \
+    ssh \
+    git \
+    libblas3 \
+    liblapack3 \
+    liblapack-dev \
+    libblas-dev \
+    build-essential \
+    ca-certificates \
+    && mkdir -p ~/.ssh \
+    && update-ca-certificates \
+    && touch ~/.ssh/known_hosts \
+    && chmod 0600 ~/.ssh/known_hosts ~/.ssh \
+    && ssh-keyscan github.com >> ~/.ssh/known_hosts
+
+COPY . /module
+WORKDIR /module
+ENV PATH="/module/.venv/bin:$PATH"
+RUN --mount=type=ssh,id=github_ssh_id set -ex \
+    && pip install poetry \
+    && poetry install --without dev --no-root --no-interaction --no-ansi --verbose \
+    && python3 -m pip install . --no-deps \
+    && apt-get remove build-essential -yqq \
+    && apt-get clean autoclean \
+    && apt-get autoremove -yqq --purge \
+    && rm -rf /var/lib/{apt,dpkg,cache,log}/ \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* \
+    && rm -rf /var/tmp/* \
+    && rm -rf ~/.cache \
+    && rm -rf /usr/share/man \
+    && rm -rf /usr/share/doc \
+    && rm -rf /usr/share/doc-base

fdmt/cpu_fdmt.py

@@ -4,7 +4,6 @@
 import numpy as np
 from time import time
 from attr import attrs, attrib, cmp_using
-#from . import fdmt_iter_par
 from .fdmt_njit import fdmt_iter_par
 @attrs
 class FDMT:
@@ -195,7 +194,6 @@ def fdmt_iteration(self, src, dest, i):
 
         fdmt_iter_par(fs, nchan, df, Qarr, src, dest, i, fmin, fmax, np.float32(maxDT), num_threads)
 
-
     def reset_ABQ(self):
         self.A = None
         self.B = None