diff --git a/brickflow/__init__.py b/brickflow/__init__.py index ba635e03..ae269277 100644 --- a/brickflow/__init__.py +++ b/brickflow/__init__.py @@ -43,9 +43,22 @@ def _insert_before_path_startswith( arr.append(new_element) +class ConfigFileType(Enum): + YAML = "yaml" + YML = "yml" + + class BrickflowProjectConstants(Enum): - DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME = ".brickflow-project-root.yml" - DEFAULT_MULTI_PROJECT_CONFIG_FILE_NAME = "brickflow-multi-project.yml" + DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME = ".brickflow-project-root" + DEFAULT_MULTI_PROJECT_CONFIG_FILE_NAME = "brickflow-multi-project" + DEFAULT_CONFIG_FILE_TYPE = ConfigFileType.YML.value + + +def get_config_file_type(brickflow_root: str) -> ConfigFileType: + for config_file_type in ConfigFileType: + if brickflow_root.endswith(config_file_type.value): + return config_file_type + return ConfigFileType.YAML class BrickflowEnvVars(Enum): @@ -339,6 +352,8 @@ def get_bundles_project_env() -> str: "BrickflowDefaultEnvs", "get_default_log_handler", "get_brickflow_version", + "ConfigFileType", + "get_config_file_type", "BrickflowProjectConstants", "BrickflowProjectDeploymentSettings", ] diff --git a/brickflow/cli/configure.py b/brickflow/cli/configure.py index 2375361d..120ec0c9 100644 --- a/brickflow/cli/configure.py +++ b/brickflow/cli/configure.py @@ -102,7 +102,10 @@ def create_entry_point(working_dir: str, data: str) -> None: def create_brickflow_project_root_marker() -> None: - path = Path(BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME.value) + path = Path( + f"{BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME.value}." + f"{BrickflowProjectConstants.DEFAULT_CONFIG_FILE_TYPE.value}" + ) if path.exists(): click.echo(f"Path: {str(path.absolute())} already exists...") # path = Path(working_dir) / "entrypoint.py.new" diff --git a/brickflow/cli/projects.py b/brickflow/cli/projects.py index 1a60aa49..47efb59c 100644 --- a/brickflow/cli/projects.py +++ b/brickflow/cli/projects.py @@ -1,6 +1,5 @@ import contextlib import os -from enum import Enum from pathlib import Path from typing import Dict, Optional, List, Generator, Any, Callable @@ -15,6 +14,8 @@ _ilog, BrickflowProjectDeploymentSettings, ctx, + ConfigFileType, + get_config_file_type, ) from brickflow.cli.bundles import ( bundle_deploy, @@ -83,11 +84,6 @@ def has_projects(self) -> bool: return self.project_roots is not None and len(self.project_roots) > 0 -class ConfigFileType(Enum): - YAML = "yaml" - JSON = "json" # unsupported - - class MultiProjectManager: def __init__( self, @@ -95,7 +91,7 @@ def __init__( file_type: ConfigFileType = ConfigFileType.YAML, ) -> None: self.file_type = file_type - self._config_file: Path = Path(config_file_name) + self._config_file: Path = Path(f"{config_file_name}.{file_type.value}") self._brickflow_multi_project_config: BrickflowMultiRootProjectConfig self._brickflow_multi_project_config = ( self._load_config() @@ -127,7 +123,10 @@ def _load_config(self) -> BrickflowMultiRootProjectConfig: return BrickflowMultiRootProjectConfig(project_roots={}) def _root_config_path(self, root: str) -> Path: - root_file = BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME.value + root_file = ( + f"{BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME.value}." + f"{BrickflowProjectConstants.DEFAULT_CONFIG_FILE_TYPE.value}" + ) return self._config_file.parent / root / root_file def _load_roots(self) -> Dict[str, BrickflowRootProjectConfig]: @@ -233,22 +232,32 @@ class BrickflowRootNotFound(Exception): def get_brickflow_root(current_path: Optional[Path] = None) -> Path: current_dir = Path(current_path or get_notebook_ws_path(ctx.dbutils) or os.getcwd()) - potential_config_file_path = ( - current_dir - / BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_CONFIG_FILE_NAME.value - ) - if potential_config_file_path.exists(): - return potential_config_file_path - elif current_dir.parent == current_dir: + + potential_config_files = [ + f"{BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME.value}.{cfg_type.value}" + for cfg_type in ConfigFileType + ] + potential_config_file_paths = [current_dir / p for p in potential_config_files] + + for potential_config_file_path in potential_config_file_paths: + if potential_config_file_path.exists(): + return potential_config_file_path + + if current_dir.parent == current_dir: # Reached the filesystem root, return just raw file value return Path( - BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_CONFIG_FILE_NAME.value + f"{BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_CONFIG_FILE_NAME.value}." + f"{BrickflowProjectConstants.DEFAULT_CONFIG_FILE_TYPE.value}" ) else: return get_brickflow_root(current_dir.parent) -multi_project_manager = MultiProjectManager(config_file_name=str(get_brickflow_root())) +brickflow_root_path = get_brickflow_root() +config_file_type = get_config_file_type(str(brickflow_root_path)) +multi_project_manager = MultiProjectManager( + config_file_name=str(brickflow_root_path), file_type=config_file_type +) def initialize_project_entrypoint( diff --git a/brickflow/resolver/__init__.py b/brickflow/resolver/__init__.py index 802ae328..4a5da1ed 100644 --- a/brickflow/resolver/__init__.py +++ b/brickflow/resolver/__init__.py @@ -37,8 +37,10 @@ def go_up_till_brickflow_root(cur_path: str) -> str: path = pathlib.Path(cur_path).resolve() valid_roots = [ - BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME.value, - BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_CONFIG_FILE_NAME.value, + f"{BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_ROOT_FILE_NAME.value}." + f"{BrickflowProjectConstants.DEFAULT_CONFIG_FILE_TYPE.value}", + f"{BrickflowProjectConstants.DEFAULT_MULTI_PROJECT_CONFIG_FILE_NAME.value}." + f"{BrickflowProjectConstants.DEFAULT_CONFIG_FILE_TYPE.value}", ] # recurse to see if there is a brickflow root and return the path