From 5de40c1550f41101fc844f56b371fbdaf70adba6 Mon Sep 17 00:00:00 2001 From: August Dahl Date: Thu, 21 Mar 2024 15:14:25 +0100 Subject: [PATCH 1/4] Oppdater dask-infrastructure-script --- scripts/dask_infrastructure.py | 99 ++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/scripts/dask_infrastructure.py b/scripts/dask_infrastructure.py index 14f38e4..19fdb08 100644 --- a/scripts/dask_infrastructure.py +++ b/scripts/dask_infrastructure.py @@ -1,39 +1,71 @@ import sys import os +import json +def edit_file(filepath, json_obj): + team_name: str = json_obj.get("team_name") + ad_group_name: str = json_obj.get("ad_group_name") + area_name: str = json_obj.get("area_name") + project_id = {} + project_id["sandbox"] = json_obj.get("project_id_sandbox") + project_id["dev"] = json_obj.get("project_id_dev") + project_id["test"] = json_obj.get("project_id_test") + project_id["prod"] = json_obj.get("project_id_prod") -def edit_file(filepath): - with open(filepath, 'r') as file: + # Define the new team data + new_team_data = generate_module_definition(ad_group_name, team_name, area_name, project_id) + + append_content_to_end_of_file(filepath, new_team_data) + +def generate_module_definition(ad_group_name: str, team_name: str, area_name: str, project_id_map: dict) -> str: + module = f''' + module "{team_name.lower()}" {{ + source = "../dbx_team_resources" + + ad_group_name = "{ad_group_name}" + team_name = "{team_name.lower()}" + area_name = "{area_name.lower()}" + deploy_sa_map = {{ + sandbox = "{team_name.lower()}-deploy@{project_id_map['sandbox']}.iam.gserviceaccount.com", + dev = "{team_name.lower()}-deploy@{project_id_map['dev']}.iam.gserviceaccount.com", + test = "{team_name.lower()}-deploy@{project_id_map['test']}.iam.gserviceaccount.com", + prod = "{team_name.lower()}-deploy@{project_id_map['prod']}.iam.gserviceaccount.com" + }} + projects = {{ + sandbox = "{project_id_map['sandbox']}", + dev = "{project_id_map['dev']}", + test = "{project_id_map['test']}", + prod = "{project_id_map['prod']}", + }} + + providers = {{ + databricks.accounts = databricks.accounts + databricks.workspace = databricks.workspace + google.global_storage = google.global_storage + }} + + metastore_id = var.metastore_id + workspace_id = var.workspace_id + workspace_env = var.workspace_env + env = var.env + all_users_group_id = var.all_users_group_id + product_teams_group_id = var.product_teams_group_id + compute_sa_teams_group_id = var.compute_sa_teams_group_id + }} + ''' + + return module + +def append_content_to_end_of_file(file_path: str, content: str) -> None: + with open(file_path) as file: lines = file.readlines() + file.close() + lines.insert(len(lines), content) + + with open(file_path, 'w') as file: + file.writelines(lines) + file.close() - # Define the new team data - new_team_data = """ - "new_team" : { - "members" : [ - "member1@example.com", - "member2@example.com" - ], - "external_users" : [], - "area_name" : "new_area", - "deploy_service_account" : { - "sandbox" : "new-team-deploy@sandbox.iam.gserviceaccount.com", - "dev" : "new-team-deploy@dev.iam.gserviceaccount.com" - }, - "project_id" : { - "sandbox" : "new-team-sandbox", - "dev" : "new-team-dev" - } - }, - """ - - # Find the end of the 'locals' block and insert the new team data - insert_index = next((i for i, line in enumerate( - reversed(lines)) if line.strip() == "},"), None) - if insert_index is not None: - lines.insert(len(lines) - insert_index, new_team_data) - - with open(filepath, 'w') as file: - file.writelines(lines) if __name__ == "__main__": @@ -42,6 +74,7 @@ def edit_file(filepath): sys.exit(1) file_path = sys.argv[1] - absolute_path = os.path.abspath(file_path) - print(f"Attempting to access: {absolute_path}") - edit_file(file_path) + json_str = sys.argv[2] + json_obj = json.loads(json_str) + + edit_file(file_path, json_obj) From ffacf9ceb562adb66458b13636b6d05633492136 Mon Sep 17 00:00:00 2001 From: August Dahl Date: Thu, 21 Mar 2024 15:29:38 +0100 Subject: [PATCH 2/4] oppdater main --- scripts/dask_infrastructure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dask_infrastructure.py b/scripts/dask_infrastructure.py index 19fdb08..9692f31 100644 --- a/scripts/dask_infrastructure.py +++ b/scripts/dask_infrastructure.py @@ -69,8 +69,8 @@ def append_content_to_end_of_file(file_path: str, content: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 3: - print("Usage: python edit_file.py ") + if len(sys.argv) < 3: + print("Usage: python script.py ") sys.exit(1) file_path = sys.argv[1] From cf55858d2e560fed4b185a9b5102283f3931a868 Mon Sep 17 00:00:00 2001 From: August Dahl Date: Thu, 21 Mar 2024 15:31:44 +0100 Subject: [PATCH 3/4] Fjern fromJson --- .github/workflows/create-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-pull-request.yml b/.github/workflows/create-pull-request.yml index 2c2d39b..8eb56cc 100644 --- a/.github/workflows/create-pull-request.yml +++ b/.github/workflows/create-pull-request.yml @@ -81,7 +81,7 @@ jobs: - name: Modify files working-directory: current-repo run: | - python scripts/${{ env.SCRIPT }} '../target-repo/${{ env.FILE_TO_MODIFY_PATH }}' '${{ fromJson(env.SCRIPT_PARAMS) }}' + python scripts/${{ env.SCRIPT }} '../target-repo/${{ env.FILE_TO_MODIFY_PATH }}' '${{ env.SCRIPT_PARAMS }}' - name: Format Terraform Files if: env.TERRAFORM_TARGET_FOLDER != null From eaa4ef7d6566ea240b0a6fb6825a06b862e4b3f6 Mon Sep 17 00:00:00 2001 From: August Dahl Date: Thu, 21 Mar 2024 15:36:31 +0100 Subject: [PATCH 4/4] Hent project_id_map --- scripts/dask_infrastructure.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/dask_infrastructure.py b/scripts/dask_infrastructure.py index 9692f31..569c043 100644 --- a/scripts/dask_infrastructure.py +++ b/scripts/dask_infrastructure.py @@ -6,14 +6,10 @@ def edit_file(filepath, json_obj): team_name: str = json_obj.get("team_name") ad_group_name: str = json_obj.get("ad_group_name") area_name: str = json_obj.get("area_name") - project_id = {} - project_id["sandbox"] = json_obj.get("project_id_sandbox") - project_id["dev"] = json_obj.get("project_id_dev") - project_id["test"] = json_obj.get("project_id_test") - project_id["prod"] = json_obj.get("project_id_prod") + project_id_map: dict = json_obj.get("project_id_map") # Define the new team data - new_team_data = generate_module_definition(ad_group_name, team_name, area_name, project_id) + new_team_data = generate_module_definition(ad_group_name, team_name, area_name, project_id_map) append_content_to_end_of_file(filepath, new_team_data)