devops stuff

- large changes to makefile - large changes to CI - move certain temp devops files to `.github`
mivanit · Jun 27, 2024 · 731dc1e · 731dc1e
1 parent 1721671
commit 731dc1e
Show file tree

Hide file tree

Showing 8 changed files with 362 additions and 337 deletions.
diff --git a/.lastversion → .github/.lastversion b/.lastversion → .github/.lastversion
diff --git a/.github/dev-requirements.txt b/.github/dev-requirements.txt
@@ -0,0 +1,94 @@
+appnope==0.1.4 ; python_version >= "3.8" and python_version < "4.0" and (platform_system == "Darwin" or sys_platform == "darwin")
+asttokens==2.4.1 ; python_version >= "3.8" and python_version < "4.0"
+backcall==0.2.0 ; python_version >= "3.8" and python_version < "4.0"
+cffi==1.16.0 ; python_version >= "3.8" and python_version < "4.0" and implementation_name == "pypy"
+click==8.1.7 ; python_version >= "3.8" and python_version < "4"
+colorama==0.4.6 ; python_version >= "3.8" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows"
+comm==0.2.2 ; python_version >= "3.8" and python_version < "4.0"
+coverage-badge==1.1.1 ; python_version >= "3.8" and python_version < "4.0"
+coverage==7.5.4 ; python_version >= "3.8" and python_version < "4.0"
+coverage[toml]==7.5.4 ; python_version >= "3.8" and python_version < "4.0"
+debugpy==1.8.2 ; python_version >= "3.8" and python_version < "4.0"
+decorator==5.1.1 ; python_version >= "3.8" and python_version < "4.0"
+exceptiongroup==1.2.1 ; python_version >= "3.8" and python_version < "3.11"
+executing==2.0.1 ; python_version >= "3.8" and python_version < "4.0"
+filelock==3.15.4 ; python_version >= "3.8" and python_version < "4.0"
+fsspec==2024.6.1 ; python_version >= "3.8" and python_version < "4.0"
+importlib-metadata==8.0.0 ; python_version >= "3.8" and python_version < "3.10"
+iniconfig==2.0.0 ; python_version >= "3.8" and python_version < "4.0"
+intel-openmp==2021.4.0 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows"
+ipykernel==6.29.4 ; python_version >= "3.8" and python_version < "4.0"
+ipython==8.12.3 ; python_version >= "3.8" and python_version < "4.0"
+jaxtyping==0.2.19 ; python_version >= "3.8" and python_version < "4.0"
+jedi==0.19.1 ; python_version >= "3.8" and python_version < "4.0"
+jinja2==3.1.4 ; python_version >= "3.8" and python_version < "4.0"
+jupyter-client==8.6.2 ; python_version >= "3.8" and python_version < "4.0"
+jupyter-core==5.7.2 ; python_version >= "3.8" and python_version < "4.0"
+libcst==1.1.0 ; python_version >= "3.8" and python_version < "4"
+markdown-it-py==3.0.0 ; python_version >= "3.8" and python_version < "4"
+markupsafe==2.1.5 ; python_version >= "3.8" and python_version < "4.0"
+matplotlib-inline==0.1.7 ; python_version >= "3.8" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.8" and python_version < "4"
+mkl==2021.4.0 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows"
+mpmath==1.3.0 ; python_version >= "3.8" and python_version < "4.0"
+muutils[array]==0.6.1 ; python_version >= "3.8" and python_version < "4.0"
+mypy-extensions==1.0.0 ; python_version >= "3.8" and python_version < "4.0"
+mypy==1.10.1 ; python_version >= "3.8" and python_version < "4.0"
+nest-asyncio==1.6.0 ; python_version >= "3.8" and python_version < "4.0"
+networkx==3.1 ; python_version >= "3.8" and python_version < "4.0"
+numpy==1.24.4 ; python_version >= "3.8" and python_version < "3.9"
+numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0"
+nvidia-cublas-cu12==12.1.3.1 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-cuda-cupti-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-cuda-nvrtc-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-cuda-runtime-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-cudnn-cu12==8.9.2.26 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-cufft-cu12==11.0.2.54 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-curand-cu12==10.3.2.106 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-cusolver-cu12==11.4.5.107 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-cusparse-cu12==12.1.0.106 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-nccl-cu12==2.20.5 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-nvjitlink-cu12==12.5.40 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+nvidia-nvtx-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.8" and python_version < "4.0"
+packaging==24.1 ; python_version >= "3.8" and python_version < "4.0"
+pandas==2.0.3 ; python_version >= "3.8" and python_version < "4.0"
+parso==0.8.4 ; python_version >= "3.8" and python_version < "4.0"
+pathspec==0.12.1 ; python_version >= "3.8" and python_version < "4"
+pexpect==4.9.0 ; python_version >= "3.8" and python_version < "4.0" and sys_platform != "win32"
+pickleshare==0.7.5 ; python_version >= "3.8" and python_version < "4.0"
+platformdirs==4.2.2 ; python_version >= "3.8" and python_version < "4.0"
+pluggy==1.5.0 ; python_version >= "3.8" and python_version < "4.0"
+prompt-toolkit==3.0.47 ; python_version >= "3.8" and python_version < "4.0"
+psutil==6.0.0 ; python_version >= "3.8" and python_version < "4.0"
+ptyprocess==0.7.0 ; python_version >= "3.8" and python_version < "4.0" and sys_platform != "win32"
+pure-eval==0.2.2 ; python_version >= "3.8" and python_version < "4.0"
+pycln==2.4.0 ; python_version >= "3.8" and python_version < "4"
+pycparser==2.22 ; python_version >= "3.8" and python_version < "4.0" and implementation_name == "pypy"
+pygments==2.18.0 ; python_version >= "3.8" and python_version < "4.0"
+pytest-cov==4.1.0 ; python_version >= "3.8" and python_version < "4.0"
+pytest==7.4.4 ; python_version >= "3.8" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.8" and python_version < "4.0"
+pytz==2024.1 ; python_version >= "3.8" and python_version < "4.0"
+pywin32==306 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.8" and python_version < "4.0"
+pyyaml==6.0.1 ; python_version >= "3.8" and python_version < "4"
+pyzmq==26.0.3 ; python_version >= "3.8" and python_version < "4.0"
+rich==13.7.1 ; python_version >= "3.8" and python_version < "4"
+ruff==0.4.10 ; python_version >= "3.8" and python_version < "4.0"
+shellingham==1.5.4 ; python_version >= "3.8" and python_version < "4"
+six==1.16.0 ; python_version >= "3.8" and python_version < "4.0"
+stack-data==0.6.3 ; python_version >= "3.8" and python_version < "4.0"
+sympy==1.12.1 ; python_version >= "3.8" and python_version < "4.0"
+tbb==2021.13.0 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows"
+tomli==2.0.1 ; python_version >= "3.8" and python_full_version <= "3.11.0a6"
+tomlkit==0.12.5 ; python_version >= "3.8" and python_version < "4"
+torch==2.3.1 ; python_version >= "3.8" and python_version < "4.0"
+tornado==6.4.1 ; python_version >= "3.8" and python_version < "4.0"
+traitlets==5.14.3 ; python_version >= "3.8" and python_version < "4.0"
+triton==2.3.1 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version < "3.12" and python_version >= "3.8"
+typeguard==4.3.0 ; python_version >= "3.8" and python_version < "4.0"
+typer==0.12.3 ; python_version >= "3.8" and python_version < "4"
+typing-extensions==4.12.2 ; python_version >= "3.8" and python_version < "4.0"
+typing-inspect==0.9.0 ; python_version >= "3.8" and python_version < "4"
+tzdata==2024.1 ; python_version >= "3.8" and python_version < "4.0"
+wcwidth==0.2.13 ; python_version >= "3.8" and python_version < "4.0"
+zipp==3.19.2 ; python_version >= "3.8" and python_version < "3.10"
diff --git a/.github/lint-requirements.txt b/.github/lint-requirements.txt
@@ -0,0 +1,17 @@
+click==8.1.7 ; python_version >= "3.8" and python_version < "4"
+colorama==0.4.6 ; python_version >= "3.8" and python_version < "4" and platform_system == "Windows"
+libcst==1.1.0 ; python_version >= "3.8" and python_version < "4"
+markdown-it-py==3.0.0 ; python_version >= "3.8" and python_version < "4"
+mdurl==0.1.2 ; python_version >= "3.8" and python_version < "4"
+mypy-extensions==1.0.0 ; python_version >= "3.8" and python_version < "4"
+pathspec==0.12.1 ; python_version >= "3.8" and python_version < "4"
+pycln==2.4.0 ; python_version >= "3.8" and python_version < "4"
+pygments==2.18.0 ; python_version >= "3.8" and python_version < "4"
+pyyaml==6.0.1 ; python_version >= "3.8" and python_version < "4"
+rich==13.7.1 ; python_version >= "3.8" and python_version < "4"
+ruff==0.4.10 ; python_version >= "3.8" and python_version < "4.0"
+shellingham==1.5.4 ; python_version >= "3.8" and python_version < "4"
+tomlkit==0.12.5 ; python_version >= "3.8" and python_version < "4"
+typer==0.12.3 ; python_version >= "3.8" and python_version < "4"
+typing-extensions==4.12.2 ; python_version >= "3.8" and python_version < "4"
+typing-inspect==0.9.0 ; python_version >= "3.8" and python_version < "4"
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -15,12 +15,52 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with: 
+          fetch-depth: 0
 
       - name: Install linters
-        run: pip install pycln isort black
+        run: make setup-format RUN_GLOBAL=1
 
       - name: Run Format Checks
-        run: make check-format
+        run: make check-format RUN_GLOBAL=1
+
+
+  check-deps:
+    name: Check dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Poetry
+        run: curl -sSL https://install.python-poetry.org | python3 -
+
+      - name: Poetry Plugins
+        run: |
+          poetry self add poetry-plugin-export
+          poetry self show plugins
+
+      - name: Check poetry.lock and .github/dev-requirements.txt
+        run: make check-dep-dev
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Install dependencies
+        run: uv pip install -r .github/dev-requirements.txt --system --no-deps # we already should have all dependencies exported into .github/dev-requirements.txt
+
+      - name: Install torch (special)
+        run: uv pip install torch==2.3.1+cpu --system --extra-index-url https://download.pytorch.org/whl/cpu
+
+      - name: Install zanj (local)
+        run: uv pip install . --system
 
   test:
     name: Test and Lint
@@ -43,20 +83,33 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with: 
+          fetch-depth: 0
 
       - name: Set up python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.versions.python }}
 
+      - name: Install uv
+        run: pip install uv
+
       - name: Install dependencies
+        # install torch first to avoid pytorch index messing things up
         run: |
-          curl -sSL https://install.python-poetry.org | python3 -
-          poetry lock --check
-          export CUDA_VISIBLE_DEVICES=0
-          poetry add torch@${{ matrix.versions.torch }}+cpu --source torch_cpu
-          poetry install --all-extras
+          uv pip install -r .github/dev-requirements.txt --system --no-deps
+          uv pip install torch==${{ matrix.versions.torch}}+cpu --system --extra-index-url https://download.pytorch.org/whl/cpu
+        
+      - name: Install zanj
+        run: uv pip install . --system
+
       - name: tests
-        run: make test
-      - name: lint
-        run: make lint
+        run: make test RUN_GLOBAL=1
+
+      # - name: tests in strict mode
+      #   # TODO: until zanj ported to 3.8 and 3.9
+      #   if: ${{ matrix.versions.python != '3.8' && matrix.versions.python != '3.9' }}
+      #   run: make test WARN_STRICT=1 RUN_GLOBAL=1
+
+      - name: check typing
+        run: make typing RUN_GLOBAL=1