diff --git a/brickflow/codegen/databricks_bundle.py b/brickflow/codegen/databricks_bundle.py index e677af6f..42c02a87 100644 --- a/brickflow/codegen/databricks_bundle.py +++ b/brickflow/codegen/databricks_bundle.py @@ -51,6 +51,7 @@ Targets, Workspace, ) +from brickflow.cli.projects import MultiProjectManager, get_brickflow_root from brickflow.codegen import ( CodegenInterface, DatabricksDefaultClusterTagKeys, @@ -461,12 +462,30 @@ def adjust_file_path(self, file_path: str) -> str: ] ).replace("//", "/") - # Finds the start position of the project name in the given file path and calculates the cut position. - # - `file_path.find(self.project.name)`: Finds the start index of the project name in the file path. - # - `+ len(self.project.name) + 1`: Moves the start position to the character after the project name. + multi_project_manager = MultiProjectManager( + config_file_name=str(get_brickflow_root()) + ) + bf_project = multi_project_manager.get_project(self.project.name) + + start_index_of_project_root = file_path.find( + bf_project.path_from_repo_root_to_project_root + ) + + if start_index_of_project_root < 0: + raise ValueError( + f"Error while adjusting file path. " + f"Project root not found in the file path: {file_path}." + ) + + # Finds the start position of the path_from_repo_root_to_project_root in the given file path + # and calculates the cut position. + # - `file_path.find: Finds the start index of the project root in the file path. + # - `+ len + 1`: Moves the start position to the character after the project root. # - Adjusts the file path by appending the local bundle path to the cut file path. cut_file_path = file_path[ - file_path.find(self.project.name) + len(self.project.name) + 1 : + start_index_of_project_root + + len(bf_project.path_from_repo_root_to_project_root) + + 1 : ] file_path = ( bundle_files_local_path + file_path diff --git a/tests/codegen/test_databricks_bundle.py b/tests/codegen/test_databricks_bundle.py index 0192ed22..a734eff6 100644 --- a/tests/codegen/test_databricks_bundle.py +++ b/tests/codegen/test_databricks_bundle.py @@ -82,6 +82,7 @@ class TestBundleCodegen(TestCase): BrickflowEnvVars.BRICKFLOW_PROJECT_TAGS.value: "tag1 = value1, tag2 =value2 ", # spaces will be trimmed }, ) + @patch("brickflow.codegen.databricks_bundle.MultiProjectManager") @patch("brickflow.engine.task.get_job_id", return_value=12345678901234.0) @patch("subprocess.check_output") @patch("brickflow.context.ctx.get_parameter") @@ -96,12 +97,16 @@ def test_generate_bundle_local( dbutils: Mock, sub_proc_mock: Mock, get_job_id_mock: Mock, + multi_project_manager_mock: Mock, ): dbutils.return_value = None sub_proc_mock.return_value = b"" bf_version_mock.return_value = "1.0.0" workspace_client = get_workspace_client_mock() get_job_id_mock.return_value = 12345678901234.0 + multi_project_manager_mock.return_value.get_project.return_value = MagicMock( + path_from_repo_root_to_project_root="test-project" + ) # get caller part breaks here with Project( "test-project", @@ -138,6 +143,7 @@ def test_generate_bundle_local( BrickflowEnvVars.BRICKFLOW_WORKFLOW_SUFFIX.value: "_suffix", }, ) + @patch("brickflow.codegen.databricks_bundle.MultiProjectManager") @patch("brickflow.engine.task.get_job_id", return_value=12345678901234.0) @patch("subprocess.check_output") @patch("brickflow.context.ctx.get_parameter") @@ -146,18 +152,23 @@ def test_generate_bundle_local( "brickflow.context.ctx.get_current_timestamp", MagicMock(return_value=1704067200000), ) + # @patch() def test_generate_bundle_local_prefix_suffix( self, bf_version_mock: Mock, dbutils: Mock, sub_proc_mock: Mock, get_job_id_mock: Mock, + multi_project_manager_mock: Mock, ): dbutils.return_value = None sub_proc_mock.return_value = b"" bf_version_mock.return_value = "1.0.0" workspace_client = get_workspace_client_mock() get_job_id_mock.return_value = 12345678901234.0 + multi_project_manager_mock.return_value.get_project.return_value = MagicMock( + path_from_repo_root_to_project_root="test-project" + ) # get caller part breaks here with Project( "test-project",