Merge pull request #7 from airflow-laminar/tkp/start

Cloning and library list utilities

airflow_common_operators/__init__.py

@@ -1,3 +1,4 @@
 from .common import *
+from .git import *
 
 __version__ = "0.1.0"

airflow_common_operators/git.py

@@ -0,0 +1,43 @@
+from typing import List
+
+from pydantic import BaseModel, Field
+
+__all__ = ("clone_repo", "GitRepo", "PipLibrary", "LibraryList")
+
+
+def clone_repo(name, repo, branch="main", install=True):
+    ret = f"""
+[[ -d {name} ]] || git clone {repo}
+pushd {name}
+git stash
+git clean -fdx
+git fetch --all --force
+git checkout {branch}
+git reset origin/{branch} --hard
+"""
+    if install:
+        ret += """
+pip install -e .
+"""
+    return ret
+
+
+class GitRepo(BaseModel):
+    name: str
+    repo: str
+    branch: str = "main"
+
+    def clone(self, install: bool = True):
+        return clone_repo(name=self.name, repo=self.repo, branch=self.branch, install=install)
+
+
+class PipLibrary(BaseModel):
+    name: str
+
+
+class LibraryList(BaseModel):
+    pip: List[PipLibrary] = Field(default_factory=list)
+    git: List[GitRepo] = Field(default_factory=list)
+
+    # def install(self):
+    #     return "pip install " + " ".join(lib.name for lib in self.libraries)

airflow_common_operators/tests/git/config/config.yaml

@@ -0,0 +1,4 @@
+# @package _global_
+_target_: airflow_config.Configuration
+defaults:
+  - extensions/[email protected]

airflow_common_operators/tests/git/config/extensions/libraries.yaml

@@ -0,0 +1,18 @@
+# @package extensions.libraries
+
+defaults: []
+
+_target_: airflow_common_operators.LibraryList
+pip:
+  - name: lib1
+  - name: lib2
+  - name: lib3
+git:
+  - name: git1
+    repo: test
+  - name: git2
+    repo: test
+    branch: develop
+  - name: git3
+    repo: test
+

airflow_common_operators/tests/git/test_git.py

@@ -0,0 +1,10 @@
+from airflow_config import load_config
+
+
+class TestConfig:
+    def test_load_config_hydra(self):
+        config = load_config(config_name="config")
+        assert config
+        assert "libraries" in config.extensions
+        assert len(config.extensions["libraries"].pip) == 3
+        assert len(config.extensions["libraries"].git) == 3

pyproject.toml

@@ -26,6 +26,7 @@ classifiers = [
 
 dependencies = [
     "apache-airflow>=2,<3",
+    "pydantic",
 ]
 
 [project.optional-dependencies]