ci: Provider Build Caching

This centralising the caching for the provider builds, along with making the
workflow a shared one.

.github/workflows/build-providers.yaml

@@ -0,0 +1,49 @@
+name: Build / Cache Providers
+
+on:
+  workflow_call:
+
+concurrency:
+  group: provider-builds
+
+jobs:
+  build-providers:
+    strategy:
+      matrix:
+        include:
+          - provider: random
+            version: 3.6.0
+            name: CallyProvidersRandom
+          - provider: google
+            version: 5.21.0
+            name: CallyProvidersGoogle
+    runs-on: ubuntu-latest
+    steps:
+      - name: Restore Provider Packages
+        id: cache-providers
+        uses: actions/cache@v4
+        with:
+          path: build/random/dist/${ matrix.name }}-${{ matrix.version }}.tar.gz
+          key: cally-provider-${{ matrix.provider }}-${{ matrix.version }}
+      - uses: actions/checkout@v4
+        if: steps.cache-providers.outputs.cache-hit != 'true'
+      - name: Setup Python
+        if: steps.cache-providers.outputs.cache-hit != 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install Cally
+        if: steps.cache-providers.outputs.cache-hit != 'true'
+        run: pip install .[provider_build]
+      - uses: actions/setup-node@v4
+        if: steps.cache-providers.outputs.cache-hit != 'true'
+        with:
+          node-version: "20"
+      - name: Install cdktf-cli and build
+        if: steps.cache-providers.outputs.cache-hit != 'true'
+        run: |
+          npm install cdktf-cli
+          echo "$(pwd)/node_modules/.bin/" >> $GITHUB_PATH
+          cally provider build --provider ${{ matrix.provider }} --version ${{ matrix.version }}
+          (cd build/${{ matrix.provider }} && python -m build)

.github/workflows/coverage-build.yaml

@@ -10,32 +10,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: .github/workflows/build-providers.yaml
+      - uses: actions/cache/restore@v3
+        id: cache
+        with:
+          path: build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
+          key: cally-provider-random-3.6.0
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: pip
-      - name: Install Cally test dependencies
-        run: pip install .[test]
-      - name: Restore Provider Packages
-        id: cache-providers
-        uses: actions/cache@v4
-        with:
-          path: build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
-          key: cally-provider-random-3.6.0
-      - uses: actions/setup-node@v4
-        if: steps.cache-providers.outputs.cache-hit != 'true'
-        with:
-          node-version: "20"
-      - name: Install cdktf-cli and build
-        if: steps.cache-providers.outputs.cache-hit != 'true'
-        run: |
-          npm install cdktf-cli
-          echo "$(pwd)/node_modules/.bin/" >> $GITHUB_PATH
-          cally provider build --provider random --version 3.6.0
-          (cd build/random && python -m build)
       - name: Install Provider Pacakge
         run: pip install build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
+      - name: Install Cally test dependencies
+        run: pip install .[test]
       - name: Run Coverage
         run: |
           coverage run -m pytest

.github/workflows/test-cally.yaml

@@ -12,35 +12,23 @@ jobs:
       matrix:
         python: ["3.11"]
     steps:
+      # Restore Provider Packages
       - uses: actions/checkout@v4
+      - uses: .github/workflows/build-providers.yaml
+      - uses: actions/cache/restore@v3
+        id: cache
+        with:
+          path: build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
+          key: cally-provider-random-3.6.0
 
-      # Cally Testing
+      # Cally Install
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
           cache: pip
       - name: Install Cally test dependencies
         run: pip install .[test]
-
-      # Provider Packages
-      - name: Restore Provider Packages
-        id: cache-providers
-        uses: actions/cache@v4
-        with:
-          path: build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
-          key: cally-provider-random-3.6.0
-      - uses: actions/setup-node@v4
-        if: steps.cache-providers.outputs.cache-hit != 'true'
-        with:
-          node-version: "20"
-      - name: Install cdktf-cli and build
-        if: steps.cache-providers.outputs.cache-hit != 'true'
-        run: |
-          npm install cdktf-cli
-          echo "$(pwd)/node_modules/.bin/" >> $GITHUB_PATH
-          cally provider build --provider random --version 3.6.0
-          (cd build/random && python -m build)
       - name: Install Provider Pacakge
         run: pip install build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
 

pyproject.toml

@@ -45,6 +45,9 @@ test = [
   "ruff",
   "types-PyYAML",
 ]
+provider_build = [
+  "build"
+]
 
 [project.scripts]
 cally = "cally.cli:cally"
@@ -53,8 +56,15 @@ cally = "cally.cli:cally"
 skip-string-normalization = true
 
 [tool.coverage.run]
+branch = true
 omit = ["tests/*"]
 
+[tool.coverage.report]
+exclude_lines = [
+    "if TYPE_CHECKING:",
+    "    pass",
+]
+
 [[tool.mypy.overrides]]
 module = "dynaconf.*"
 ignore_missing_imports = true

src/cally/cli/config/loader.py

@@ -3,6 +3,11 @@
 import yaml
 from dynaconf import LazySettings
 
+try:
+    from cally.idp.defaults import DEFAULTS as IDP_DEFAULTS  # type: ignore
+except ModuleNotFoundError:
+    IDP_DEFAULTS: dict = {}  # type: ignore[no-redef]
+
 
 def load(obj: LazySettings, *args, **kwargs) -> None:  # noqa: ARG001
     """
@@ -31,6 +36,7 @@ def load(obj: LazySettings, *args, **kwargs) -> None:  # noqa: ARG001
         loaded = yaml.safe_load(config_file.read_text())
 
     # Defaults
+    obj.update(IDP_DEFAULTS)
     obj.update(loaded.get('defaults', {}))
     if obj.cally_env is not None:
         obj.update(environment=obj.cally_env)

tests/__init__.py

@@ -1,5 +1,7 @@
 import json
 import os
+import sys
+from importlib import import_module, reload
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from unittest import TestCase, mock
@@ -56,3 +58,35 @@ def synth_stack(self, stack: CallyStack) -> dict:
         stack.synth_stack(self.working.name)
         output_file = Path(self.working.name, 'stacks', stack.name, 'cdk.tf.json')
         return json.loads(output_file.read_text())
+
+
+class CallyIdpTestHarness(CallyTfTestHarness):
+    def setUp(self):
+        sys.path.append(Path(Path(__file__).parent, 'testdata/test_cls').as_posix())
+        self.reload_cally_modules()
+        super().setUp()
+
+    def tearDown(self):
+        super().tearDown()
+        sys.path.remove(Path(Path(__file__).parent, 'testdata/test_cls').as_posix())
+        for mod in [
+            'cally.idp',
+            'cally.idp.defaults',
+            'cally.idp.stacks',
+            'cally.idp.commands',
+        ]:
+            if mod in sys.modules:
+                del sys.modules[mod]
+        self.reload_cally_modules()
+
+    @staticmethod
+    def reload_cally_modules():
+        modules = [
+            'cally.cli',
+            'cally.cdk.stacks',
+            'cally.cli.config.loader',
+            'cally.cli.config',
+        ]
+        for module in modules:
+            mod = import_module(module)
+            reload(mod)

tests/cli/config/__init__.py

@@ -1,10 +1,10 @@
 import os
 from unittest import mock
 
-from cally.cli.config import CallyConfig
-from cally.cli.config import types as cally_types
 from dynaconf import ValidationError
 
+from cally.cli.config import CallyConfig
+from cally.cli.config import types as cally_types
 from ... import CallyTestHarness
 
 

tests/cli/config/test_idp.py

@@ -0,0 +1,17 @@
+from cally.cli.config import CallyConfig
+
+from ... import CallyIdpTestHarness
+
+
+class CallyIdpConfigTests(CallyIdpTestHarness):
+
+    def test_idp_defaults(self):
+        config = CallyConfig(config_file='blah.yaml')
+        config.environment = 'harness'
+        config.service = 'idp-defaults-ya'
+        data = {
+            'ENVIRONMENT': 'harness',
+            'NAME': 'idp-defaults-ya',
+            'PROVIDERS': {'provido': {'location': 'some-place1'}},
+        }
+        self.assertDictEqual(config.settings.to_dict(), data)

tests/cli/test_config.py

@@ -10,7 +10,6 @@
 
 
 class ConfigCliTests(CallyTestHarness):
-
     @mock.patch.dict(os.environ, {"CALLY_STACK_TYPE": "CallyStack"})
     def test_print_service_basic(self):
         result = CliRunner().invoke(

tests/cli/test_idp.py

@@ -1,20 +1,11 @@
-import sys
-from importlib import reload
-from pathlib import Path
-
-sys.path.append(Path(Path(__file__).parent, '../testdata/test_cls').as_posix())
-
 from cally import cli
 from cally.cli.config import CallyConfig
 from click.testing import CliRunner
 
-from .. import CallyTestHarness
+from .. import CallyIdpTestHarness
 
 
-class CliTests(CallyTestHarness):
-    def setUp(self):
-        super().setUp()
-        reload(cli)
+class CliTests(CallyIdpTestHarness):
 
     def test_example(self):
         result = CliRunner().invoke(

tests/stacks/test_idp.py

@@ -1,19 +1,9 @@
-import sys
-from importlib import reload
-from pathlib import Path
-
-sys.path.append(Path(Path(__file__).parent, '../testdata/test_cls').as_posix())
-
 from cally.cdk import stacks
 
-from .. import CallyTfTestHarness
-
+from .. import CallyIdpTestHarness
 
-class IdpStackTests(CallyTfTestHarness):
-    def setUp(self):
-        super().setUp()
-        reload(stacks)
 
+class IdpStackTests(CallyIdpTestHarness):
     def test_class_load(self):
         self.assertEqual(stacks.MinimalStack.__name__, 'MinimalStack')
 

tests/testdata/test_cls/cally/idp/defaults.py

@@ -0,0 +1 @@
+DEFAULTS = {'providers': {'provido': {'location': 'some-place1'}}}