From 422e75bb505d2aa46e56a88608eec311bd475390 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 14 Aug 2024 13:11:32 -0700 Subject: [PATCH 01/39] Microsoft Advertising (Bing Ads) --- .../config/microsoft_advertising_config.yml | 93 +++++++++++++++++++ .../dataset/microsoft_advertising_dataset.yml | 5 + ...microsoft_advertising_request_overrides.py | 26 ++++++ .../saas/microsoft_advertising_fixtures.py | 37 ++++++++ .../saas/test_microsoft_advertising_task.py | 28 ++++++ 5 files changed, 189 insertions(+) create mode 100644 data/saas/config/microsoft_advertising_config.yml create mode 100644 data/saas/dataset/microsoft_advertising_dataset.yml create mode 100644 src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py create mode 100644 tests/fixtures/saas/microsoft_advertising_fixtures.py create mode 100644 tests/ops/integration_tests/saas/test_microsoft_advertising_task.py diff --git a/data/saas/config/microsoft_advertising_config.yml b/data/saas/config/microsoft_advertising_config.yml new file mode 100644 index 0000000000..cc19828336 --- /dev/null +++ b/data/saas/config/microsoft_advertising_config.yml @@ -0,0 +1,93 @@ +saas_config: + fides_key: + name: Microsoft Advertising + type: microsoft_advertising + description: A sample schema representing the Microsoft Advertising integration for Fides + user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/microsoft-advertising + version: 0.1.0 + + connector_params: + - name: domain + default_value: clientcenter.api.sandbox.bingads.microsoft.com # change to the prod URL for the default + description: The base URL for your Microsoft Advertising + - name: client_id + label: Client ID + description: Your Microsoft Advertising application's client ID + - name: client_secret + label: Client secret + description: Your Microsoft Advertising application's client secret + sensitive: True + - name: redirect_uri + label: Redirect URL + description: The Fides URL to which users will be redirected upon successful authentication + + client_config: + protocol: https + host: + authentication: + strategy: oauth2_authorization_code + configuration: + authorization_request: + method: GET + client_config: + protocol: https + host: login.windows-ppe.net + path: /consumers/oauth2/v2.0/authorize + query_params: + - name: client_id + value: + - name: redirect_uri + value: + - name: response_type + value: code + - name: scope + value: openid profile offline_access https://ads.microsoft.com/msads.manage + - name: state + value: + token_request: + method: POST + client_config: + protocol: https + host: login.windows-ppe.net + path: /consumers/oauth2/v2.0/token + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + body: | + { + "client_id": "", + "client_secret": "", + "grant_type": "authorization_code", + "code": "", + "redirect_uri": "" + } + refresh_request: + method: POST + client_config: + protocol: https + host: login.windows-ppe.net + path: /consumers/oauth2/v2.0/token + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + body: | + { + "client_id": "", + "client_secret": "", + "grant_type": "refresh_token", + "refresh_token": "", + "redirect_uri": "" + } + + test_request: + method: GET + path: /api/rest/v6/users # update this with a simple API call that uses the token, just to verify + + endpoints: + - name: user + requests: + delete: + request_override: microsoft_advertising_user_delete + param_values: + - name: email + identity: email diff --git a/data/saas/dataset/microsoft_advertising_dataset.yml b/data/saas/dataset/microsoft_advertising_dataset.yml new file mode 100644 index 0000000000..9cb38cf752 --- /dev/null +++ b/data/saas/dataset/microsoft_advertising_dataset.yml @@ -0,0 +1,5 @@ +dataset: + - fides_key: + name: Microsoft Advertising Dataset + description: A sample dataset representing the Microsoft Advertising integration for Fides + collections: [] diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py new file mode 100644 index 0000000000..da30b98048 --- /dev/null +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -0,0 +1,26 @@ +from typing import Any, Dict, List + +from fides.api.models.policy import Policy +from fides.api.models.privacy_request import PrivacyRequest +from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient +from fides.api.service.saas_request.saas_request_override_factory import ( + SaaSRequestType, + register, +) + + +@register("microsoft_advertising_user_delete", [SaaSRequestType.DELETE]) +def microsoft_advertising_user_delete( + client: AuthenticatedClient, + param_values_per_row: List[Dict[str, Any]], + policy: Policy, + privacy_request: PrivacyRequest, + secrets: Dict[str, Any], +) -> int: + rows_updated = 0 + + for row_param_values in param_values_per_row: + # API calls go here, look at other request overrides in this directory as examples + rows_updated += 1 + + return rows_updated diff --git a/tests/fixtures/saas/microsoft_advertising_fixtures.py b/tests/fixtures/saas/microsoft_advertising_fixtures.py new file mode 100644 index 0000000000..18e1520395 --- /dev/null +++ b/tests/fixtures/saas/microsoft_advertising_fixtures.py @@ -0,0 +1,37 @@ +from typing import Any, Dict + +import pydash +import pytest + +from tests.ops.integration_tests.saas.connector_runner import ( + ConnectorRunner, + generate_random_email, +) +from tests.ops.test_helpers.vault_client import get_secrets + +secrets = get_secrets("microsoft_advertising") + + +@pytest.fixture(scope="session") +def microsoft_advertising_secrets(saas_config) -> Dict[str, Any]: + return { + "domain": pydash.get(saas_config, "microsoft_advertising.domain") + or secrets["domain"] + # add the rest of your secrets here + } + + +@pytest.fixture +def microsoft_advertising_erasure_identity_email() -> str: + return generate_random_email() + + +@pytest.fixture +def microsoft_advertising_runner( + db, + cache, + microsoft_advertising_secrets, +) -> ConnectorRunner: + return ConnectorRunner( + db, cache, "microsoft_advertising", microsoft_advertising_secrets + ) diff --git a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py new file mode 100644 index 0000000000..30fd53f363 --- /dev/null +++ b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py @@ -0,0 +1,28 @@ +import pytest + +from fides.api.models.policy import Policy +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner + + +@pytest.mark.integration_saas +class TestMicrosoftAdvertisingConnector: + def test_connection(self, microsoft_advertising_runner: ConnectorRunner): + microsoft_advertising_runner.test_connection() + + async def test_non_strict_erasure_request( + self, + microsoft_advertising_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + microsoft_advertising_erasure_identity_email: str, + microsoft_advertising_erasure_data, + ): + ( + _, + erasure_results, + ) = await microsoft_advertising_runner.non_strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": microsoft_advertising_erasure_identity_email}, + ) + assert erasure_results == {"microsoft_advertising_instance": 1} From aaaad5fe22c158e63e70292bd0cc38f84872f8b8 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 14 Aug 2024 16:31:50 -0400 Subject: [PATCH 02/39] Adding Base Bing script placeholder space --- src/playground/bings_script.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/playground/bings_script.py diff --git a/src/playground/bings_script.py b/src/playground/bings_script.py new file mode 100644 index 0000000000..d2d59e2374 --- /dev/null +++ b/src/playground/bings_script.py @@ -0,0 +1,2 @@ + +## Placeholder for the Bings Ads Script From 3fc13c020c232aa99642c98c901043d57503218c Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 20 Aug 2024 10:13:53 -0400 Subject: [PATCH 03/39] Base Bing Script. Using Sanbox credentials. TBD: Set up the correct architecture on fides for the script --- src/playground/bings_script.py | 224 +++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/src/playground/bings_script.py b/src/playground/bings_script.py index d2d59e2374..5bedb088c3 100644 --- a/src/playground/bings_script.py +++ b/src/playground/bings_script.py @@ -1,2 +1,226 @@ ## Placeholder for the Bings Ads Script +### Bing Script + +## Programatically getting the basic for this Bing Ads Call + +## V0.1 + +### Required packages: +import requests +import csv +import hashlib +import shutil +import json +### IMPORTANT: The Base XML package of python is prone to vulnerabilities. I.E Billion Laughs Bomb. Using safer defusedxml package +from defusedxml.ElementTree import fromstring + +### Functions Setup + +## Why do we put the payload on one line: Payloads Fail with line break. + + +## Step 1: Retrieval of User ID + +# Traverses the GetUserRequest SOAP XML tree response and retrieves the User ID +def getUserIdFromResponse(xmlRoot): + ## TODO: Check if we can avoid this Nesting mess + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}GetUserResponse"): + for subleaf in leaf: + if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}User"): + for user_leaf in subleaf: + if(user_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): + return user_leaf.text + +# Container Function of the Get User Process +def callGetUserRequestAndRetrieveUserId(developer_token, authentication_token): + customer_manager_service_url = "https://clientcenter.api.sandbox.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" + payload = "\n \n " + developer_token + "\n " + authentication_token + "\n \n \n \n \n \n \n" + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'GetUser' + } + + response = requests.request("POST", customer_manager_service_url, headers=headers, data=payload) + + #print( "#####////######") + #print(response.text) + #print( "#####////######") + + return getUserIdFromResponse(fromstring(response.text)) + +## Step 2 : Retrieve the Account +## TODO: What happens if the base account has multiple accounts? + +def getAccountIdFromResponse(xmlRoot): + ## TODO: Check if we can avoid this Nesting mess + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}SearchAccountsResponse"): + for subleaf in leaf: + ## TODO: Expand for Multiple accounts + if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}Accounts"): + for account_leaf in subleaf: + if(account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}AdvertiserAccount"): + for ads_account_leaf in account_leaf: + if(ads_account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): + return ads_account_leaf.text + +def callGetAccountRequestAndRetrieveAccountId(developer_token, authentication_token, user_id): + customer_manager_service_url = "https://clientcenter.api.sandbox.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" + payload = "\n\n \n \n "+ authentication_token + "\n " + developer_token + "\n \n \n \n \n \n UserId\n Equals\n "+ user_id +"\n \n \n \n \n 0\n 10\n \n \n \n\n" + + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'SearchAccounts' + } + + response = requests.request("POST", customer_manager_service_url, headers=headers, data=payload) + + #print(response.text) + + return getAccountIdFromResponse(fromstring(response.text)) + +### Step 3 + +def getAudiencesIDFromLeaf(xmlLeaf): + ## TODO: Check if we can avoid this Nesting mess + audience_ids = [] + for subleaf in xmlLeaf: + ## TODO: Expand for Multiple accounts + if(subleaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audiences"): + for audience_leaf in subleaf: + if(audience_leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audience"): + for audience_entity in audience_leaf: + if(audience_entity.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Id"): + audience_ids.append(audience_entity.text) + break + + return audience_ids + + +def getAudiencesIDsfromResponse(xmlRoot): + ## TODO: Check if we can avoid this Nesting mess + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetAudiencesByIdsResponse"): + return getAudiencesIDFromLeaf(leaf) + + +def callGetCustomerListAudiencesByAccounts(developer_token, authentication_token, user_id, account_id): + campaing_manager_service_url = "https://campaign.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" + payload = "\n \n GetAudiencesByIds\n " + authentication_token + "\n " + account_id + "\n " + user_id + "\n " + developer_token + "\n \n \n \n CustomerList\n \n \n\n" + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'GetAudiencesByIds', + } + + response = requests.request("POST", campaing_manager_service_url, headers=headers, data=payload) + return getAudiencesIDsfromResponse(fromstring(response.text)) + +### Step 4 > Build the CSV +def createCSVForRemovingCustomerListObject(audiences_ids,target_email): + base_filepath = "fixtures/CustomerListRemoval.csv" + destination = "CustomerListRemoval.csv" + csv_headers = ["Type","Status","Id","Parent Id","Client Id","Modified Time","Name","Description","Scope","Audience","Action Type","SubType","Text"] + print("Target Email to remove:" + target_email) + ## Hash the Email + hashedEmail=hashlib.sha256(target_email.encode()).hexdigest() + print(" Hashed Email>> " + hashedEmail) + ## Copy the Fixture File + shutil.copyfile(base_filepath, destination) + + with open(destination,'a') as csvfile: + writer = csv.DictWriter(csvfile,csv_headers) + for audience_id in audiences_ids: + writer.writerow({"Type": "Customer List", "Id": audience_id, "Client Id": "fides_ethyca", "Action Type": "Update" }) + writer.writerow({"Type": "Customer List Item", "Parent Id": audience_id, "Action Type": "Delete", "SubType": "Email", "Text": hashedEmail}) + +### Step 5 > Get the Bulk Upload URL + +def getUploadURLFromResponse(xmlRoot): + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetBulkUploadUrlResponse"): + for subleaf in leaf: + if(subleaf.tag== "{https://bingads.microsoft.com/CampaignManagement/v13}UploadUrl"): + return subleaf.text + +def getBulkUploadURL(developer_token, authentication_token, user_id, account_id): + bulk_api_url = "https://bulk.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc?wsdl" + + payload = "\n \n GetBulkUploadUrl\n " + authentication_token + "\n " + account_id + "\n " + user_id +"\n "+ developer_token + "\n \n \n \n ErrorsAndResults\n " + account_id + "\n \n \n\n" + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'GetBulkUploadUrl', + } + + response = requests.request("POST", bulk_api_url, headers=headers, data=payload) + + return getUploadURLFromResponse(fromstring(response.text)) + +### Step 6 : Upload to the API + +def bulkUploadCustomerList(url,developer_token, authentication_token, user_id, account_id ): + + url = "https://fileupload.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/FileUpload/File/UploadBulkFile/4bba40a0-3f2c-4242-ab08-3c166e833586?Ver=13" + + payload = {'AuthenticationToken': authentication_token, + 'DeveloperToken': developer_token, + 'CustomerId': user_id, + 'AccountId': account_id} + files=[ + ('uploadFile',('customerlist.csv',open('CustomerListRemoval.csv','rb'),'application/octet-stream')) + ] + + headers = { + 'Authorization': 'Bearer' + authentication_token + } + + response = requests.request("POST", url, headers=headers, data=payload, files=files) + + return json.loads(response.text) + + +### Variables Setup +## On the Bings Sandbox enviroment, we dont need to specify the Dev token, since its always the same Token +## Outside Sanbox, we would need to set up the Flow to get the developer token +# Oh what that means we need to do it, dosent it? /s + +sandbox_developer_token = "BBD37VB98" +current_authentication_token = "eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhDQkMtSFMyNTYiLCJ4NXQiOiJJekQ3cTdrcUowdzU0R0wxVUJtT3FHMU9JS0kiLCJ6aXAiOiJERUYifQ.KMko1d7qFCrt65CypzFmuCqt25oHhEhmiie1lZV_FxeP8HzbOjBIW4oJHxwRSQ79SIGn-Ngi6z-32uraZW_cNgBv0UrGWqxOEbhTl6qluDOjP3qiKGu0qGz3tOspjsr6EGXf0118qHXx3Tvo2KrwCPNayUMU7PKsBsm9XuLd2NoTc5-f2gRd-o6-p1-4Zo91mwpRA-FemZpgjFHhH5EYmwgOTwAC9nqLLqtIuuzqjXIkdDLSwxbkw9tINDtrgBjLcijDl3shfWGkpYYgeilP0b56lQUMQq07P7AVzXk8r7KGr_KNUDsRM24MBgCbGJeLRQzCHdu0A2BcpqwwEwkI6A.SfQdsBdwtOC3aBQbh-dLPQ.iTaeopzgt17VAbAKf6H2eyIguyFmqVB3iH2rFTNn-yHHBCh3VJOa4X7NK9bz4NMEZvDZvCoJZz7BquD3H6UPDesk-g_mNF3sS94S1pKg9TgVYQytRJBpjNC-8UmFEpKORK67et2i-GTUYHNzlOsrp5E26WArbeT7RJmxOW7iYh9hrp1Fkk7_87ofi3gHoHmrsaKBQqMnJFnNWHvAzaSz2nExfyl1t5zvg4MxgysgvPsdO5QQIcwlYCTSLfEztqG57cZ1TDOE-QbwwIh8ocCb435ENvi9qXvULcN10K-FMtTXKu96wvPV_fLx2q0B_ohixRIQtronXRfr1ZYGMhsZ02gNnl3IZuDcAobUAv_QfxJJgC7Je311ouMv9zC23-hI7poPYHUgHxWvziaz_vmRqMu2X90bEGU7XL5QECUM8W1uPl1jYzlGYZsjMj2gxkJ8UvIx3euOW3l1_ZTV5xShh9UxBrx66mwYvApO8WmGc8IYeCg7KmgUvVZmIF03rW5feK8PCzt84HsAMqU9shCcLWmhrByoB53ew7LuzjPIEB5GG-ZphqLdDSMp4Wr_u0_-mjZWxVK0Vtk2kCK6WgXjT0fgpkAxj5kH4SvtzF5L6l3Gu9B-VtF2ywoy9Ec4065p-Fi3vAoaAs6RPWMlp0Oc05PErawv1P1myotAb9QocRbhHQwD2LmX5_1vlIQ2lfxh6TXYk64NMqr3bYsMiG2M7PXC9JpgH-FbgM7LVgUDKYxArBPg3gi5vGbbSL7gyVbhAJgDwZzyYHLkUzhavS2NpUTyxdrm0cPzvXa9R130Q3AjXplbm4DZv03Y9r9sO8IM0HYMZBzkL4RTKV98DUIIe89_SOdjA8pZk1DG86KleOjyk9-CXJTWiZZP8wurNrtZ3Z4bryaagQbg-AkI4AjsrGrwcTFpWIdLXm4p98t0ByYcUdnrH3z31O45CWeQXacQOQk1tD6ixqWS1Hikkv8CoAfxwQvRSWNiM2UmNnxp28i2t30cEg2e2wN9V3xSDaftwcE1dyrKKryr4Ql37Wbv7GfSE44MAFyoNaJpjUx59L98dbPGfTLOM81gwC2arW8DYsSHmsctYTea-EwSKpwGRbbQyLf1LDd0YgADAk63X9GMyIBhehQp9Z99jglt1HStmiAsNpQno276_ndCdJzEtlnErYNCZ1JaUjffzPn7R1rztltadjlOCppC0VdqwxyAHNiH0EL0JSeQcKzbfUUzeC_5iYUJrAMcDsSwB9cqh2QX2v4gljKeN1gWUGGzWWQ3rxj2SH8abJHFdqB1tT9OxFSeqFbTDZxp_qces0Xs-2Vnj2xfRofab88hyVQ_tgN-XsXY9PservIoDA1_RhLytiMdmy6e0HevBWr421X_PktzHTZW5IR3XiZOBcNLEa9kEGkZTkx8bWnrVJlyLTqUemo6yPZzJ5hdCvDZvoJrcz807mibp3MywjVke2xyt0xDVnV7hw7WAEor4uEekE_6PA.YLzROApiy8XWW9BKmGwLYg" + +print("Call numero One ######") +user_id = callGetUserRequestAndRetrieveUserId(sandbox_developer_token, current_authentication_token) +print("Gathered User Id:",user_id) + +print("Call Numbe Dos ######") +account_id = callGetAccountRequestAndRetrieveAccountId(sandbox_developer_token, current_authentication_token, user_id) +print("Gathered Account Id:",account_id) + +print ("Llamada Number 3") +audience_ids = callGetCustomerListAudiencesByAccounts(sandbox_developer_token,current_authentication_token, user_id, account_id) +print("Gathered Audience IDs:") +print(audience_ids) + +print("Building the CSV") +createCSVForRemovingCustomerListObject(audience_ids,"someemail@data.cl") + +print("Llamada 4, Obteniendo el Bulk Upload URL") + +upload_url = getBulkUploadURL(sandbox_developer_token,current_authentication_token,user_id,account_id) + +print("Upload URL: "+ upload_url) + +print("Llamada 5: Uploading Customer List File") +response = bulkUploadCustomerList(upload_url,sandbox_developer_token,current_authentication_token,user_id,account_id) +print(response["TrackingId"]) + + + From 5cdec3c03107596c72c10d8cf8582577ce15dacf Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 20 Aug 2024 10:15:48 -0400 Subject: [PATCH 04/39] Adding fixture csv for Bulk Uploading --- src/playground/fixtures/CustomerListRemoval.csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/playground/fixtures/CustomerListRemoval.csv diff --git a/src/playground/fixtures/CustomerListRemoval.csv b/src/playground/fixtures/CustomerListRemoval.csv new file mode 100644 index 0000000000..7ff5deee83 --- /dev/null +++ b/src/playground/fixtures/CustomerListRemoval.csv @@ -0,0 +1,2 @@ +Type,Status,Id,Parent Id,Client Id,Modified Time,Name,Description,Scope,Audience,Action Type,SubType,Text +Format Version,,,,,,6.0,,,,,, \ No newline at end of file From 41c9053497f26d5d3b82c01d633cce4dcb3f935e Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 20 Aug 2024 16:17:57 -0400 Subject: [PATCH 05/39] Setting up Call Nro 1 --- ...microsoft_advertising_request_overrides.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index da30b98048..e41047cea7 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -1,12 +1,45 @@ from typing import Any, Dict, List +from defusedxml import ElementTree +from loguru import logger from fides.api.models.policy import Policy from fides.api.models.privacy_request import PrivacyRequest +from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient from fides.api.service.saas_request.saas_request_override_factory import ( SaaSRequestType, register, ) +from fides.api.util.logger_context_utils import request_details + + +sandbox_customer_manager_service_url = "https://clientcenter.api.sandbox.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" +sandbox_campaing_manager_service_url = "https://campaign.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" +sandbox_bulk_api_url = "https://bulk.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc?wsdl" + + +@register("microsoft_advertising_test_connection", [SaaSRequestType.TEST]) +def microsoft_advertising_test_connection( + client: AuthenticatedClient, + param_values_per_row: List[Dict[str, Any]], + policy: Policy, + privacy_request: PrivacyRequest, + secrets: Dict[str, Any], + is_sandbox: bool = False +) -> int: + rows_updated = 0 + + access_token = secrets["access_token"] + dev_token = secrets["dev_token"] + + for row_param_values in param_values_per_row: + # API calls go here, look at other request overrides in this directory as examples + + + rows_updated += 1 + + return rows_updated + @register("microsoft_advertising_user_delete", [SaaSRequestType.DELETE]) @@ -16,11 +49,85 @@ def microsoft_advertising_user_delete( policy: Policy, privacy_request: PrivacyRequest, secrets: Dict[str, Any], + is_sandbox: bool = False ) -> int: rows_updated = 0 + access_token = secrets["access_token"] + dev_token = secrets["dev_token"] + for row_param_values in param_values_per_row: # API calls go here, look at other request overrides in this directory as examples + user_id = callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) + + + ## llamada 2 + + ## Llamada 3 + + ## Llamada 4 + + ## Build Up del csv + + ## Llamada 5. Enviar el CSv + + + cliet.send(SaasRequestParams()) rows_updated += 1 return rows_updated + +def getUserIdFromResponse(xmlRoot): + """ + Retrieves the ID from the expected XML response of the GetUserRequest + """ + ## TODO: Check if we can avoid this Nesting mess + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}GetUserResponse"): + for subleaf in leaf: + if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}User"): + for user_leaf in subleaf: + if(user_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): + return user_leaf.text + + +def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_token: str, authentication_token: str): + """ + Calls the GetUserRequest SOAP endpoint and retrieves the User ID from the response + """ + + payload = "\n \n " + developer_token + "\n " + authentication_token + "\n \n \n \n \n \n \n" + + client.client_config.host = sandbox_customer_manager_service_url + + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'GetUser' + } + + request_params = SaaSRequestParams( + method=HTTPMethod.POST, + path="", + headers=headers, + body=payload + ) + response = client.send( + request_params + ) + + context_logger = logger.bind( + **request_details( + client.get_authenticated_request(request_params), response + ) + ) + + user_id = getUserIdFromResponse(ElementTree.fromstring(response.text)) + + if user_id is None: + context_logger.error( + "GetUser request failed with the following message {}.", response.text + ) + + return user_id From f0412d7680e720b1f11420b9e8516d11e7050ad3 Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 20 Aug 2024 16:34:42 -0400 Subject: [PATCH 06/39] Updating Call Nro 2, Getting Account Request --- ...microsoft_advertising_request_overrides.py | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index e41047cea7..3544d1d6fa 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -58,10 +58,10 @@ def microsoft_advertising_user_delete( for row_param_values in param_values_per_row: # API calls go here, look at other request overrides in this directory as examples - user_id = callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) + user_id = callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) - ## llamada 2 + account_id = callGetAccountRequestAndRetrieveAccountId(client, dev_token, access_token, user_id) ## Llamada 3 @@ -131,3 +131,62 @@ def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_ ) return user_id + +def getAccountIdFromResponse(xmlRoot): + """ + Retrieves the ID from the expected XML response of the SearchAccountsRequest + TODO: Expand for Multiple accounts + """ + ## TODO: Check if we can avoid this Nesting mess + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}SearchAccountsResponse"): + for subleaf in leaf: + ## TODO: Expand for Multiple accounts Here + if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}Accounts"): + for account_leaf in subleaf: + if(account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}AdvertiserAccount"): + for ads_account_leaf in account_leaf: + if(ads_account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): + return ads_account_leaf.text + +def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , developer_token: str, authentication_token: str, user_id: str): + """ + Calls the SearchAccounts SOAP endpoint and retrieves the Account ID from the response + """ + + payload = "\n\n \n \n "+ authentication_token + "\n " + developer_token + "\n \n \n \n \n \n UserId\n Equals\n "+ user_id +"\n \n \n \n \n 0\n 10\n \n \n \n\n" + + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'SearchAccounts' + } + + client.client_config.host = sandbox_customer_manager_service_url + + request_params = SaaSRequestParams( + method=HTTPMethod.POST, + path="", + headers=headers, + body=payload + ) + + response = client.send( + request_params + ) + + context_logger = logger.bind( + **request_details( + client.get_authenticated_request(request_params), response + ) + ) + + accountId = getAccountIdFromResponse(ElementTree.fromstring(response.text)) + + if accountId is None: + context_logger.error( + "SearchAccounts request failed with the following message {}.", response.text + ) + + return accountId From 3d2a009fb4e37a9a6c00ee3a3ab5f08ed4c8f530 Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 20 Aug 2024 17:23:18 -0400 Subject: [PATCH 07/39] Updating Call Nro 3: Getting Audiences by ID Considering ifan empty Audience List is a Fail State or not. We might not want to raise an Exception on these case, but fail silently? To consider. Leaving a Comment --- ...microsoft_advertising_request_overrides.py | 78 ++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 3544d1d6fa..48b38f7838 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -77,7 +77,7 @@ def microsoft_advertising_user_delete( return rows_updated -def getUserIdFromResponse(xmlRoot): +def getUserIdFromResponse(xmlRoot: ElementTree.Element): """ Retrieves the ID from the expected XML response of the GetUserRequest """ @@ -132,7 +132,7 @@ def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_ return user_id -def getAccountIdFromResponse(xmlRoot): +def getAccountIdFromResponse(xmlRoot: ElementTree.Element): """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts @@ -190,3 +190,77 @@ def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , deve ) return accountId + + +def getAudiencesIDFromLeaf(xmlLeaf: ElementTree.Element): + """ + Gets the Audiences from the XML Node extracted from the GetAudiencesByIdsResponse + """ + ## TODO: Check if we can avoid this Nesting mess + audience_ids = [] + for subleaf in xmlLeaf: + ## TODO: Expand for Multiple accounts having the same Audiences + if(subleaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audiences"): + for audience_leaf in subleaf: + if(audience_leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audience"): + for audience_entity in audience_leaf: + if(audience_entity.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Id"): + audience_ids.append(audience_entity.text) + break + + return audience_ids + + +def getAudiencesIDsfromResponse(xmlRoot:ElementTree.Element): + """ + Gets the Audience Leaf nodes from the GetAudiencesByIdsResponse + """ + ## TODO: Check if we can avoid this Nesting mess + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetAudiencesByIdsResponse"): + return getAudiencesIDFromLeaf(leaf) + + +def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, developer_token:str, authentication_token:str , user_id:str, account_id: str): + payload = "\n \n GetAudiencesByIds\n " + authentication_token + "\n " + account_id + "\n " + user_id + "\n " + developer_token + "\n \n \n \n CustomerList\n \n \n\n" + + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'GetAudiencesByIds', + } + + client.client_config.host = sandbox_campaing_manager_service_url + + request_params = SaaSRequestParams( + method=HTTPMethod.POST, + path="", + headers=headers, + body=payload + ) + + response = client.send( + request_params + ) + + context_logger = logger.bind( + **request_details( + client.get_authenticated_request(request_params), response + ) + ) + + + response = client.send( + request_params + ) + + audiences_list = getAudiencesIDsfromResponse(ElementTree.fromstring(response.text)) + + if not audiences_list: + ## Caveat: Do we want to throw error when the Audiences is empty? + context_logger.error( + "GetAudiencesByIds collected No audiences {}.", response.text + ) + + return audiences_list From 2723fc3757044cc317b2cc6e1336720cbd510f0f Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 20 Aug 2024 17:34:41 -0400 Subject: [PATCH 08/39] Building up the CSV Moving the Fixture CSV to the proper Location --- ...microsoft_advertising_request_overrides.py | 44 +++++++++++++++++-- .../CustomerListRemoval.csv | 0 2 files changed, 40 insertions(+), 4 deletions(-) rename src/{playground/fixtures => fides/api/service/saas_request/upload_files_templates}/CustomerListRemoval.csv (100%) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 48b38f7838..109ff86518 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -1,3 +1,7 @@ +import hashlib +import shutil +import csv + from typing import Any, Dict, List from defusedxml import ElementTree from loguru import logger @@ -51,23 +55,32 @@ def microsoft_advertising_user_delete( secrets: Dict[str, Any], is_sandbox: bool = False ) -> int: + """ + Process of removing an User email from the Microsoft Advertising Platform + + Gets the User ID, Account ID and Audiences List from the Microsoft Advertising API + Builds up the CSV for removing the Customer List Object from the Audiences + And finally gets the Bulk Upload URL to send the CSV + """ rows_updated = 0 access_token = secrets["access_token"] dev_token = secrets["dev_token"] for row_param_values in param_values_per_row: - # API calls go here, look at other request overrides in this directory as examples user_id = callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) account_id = callGetAccountRequestAndRetrieveAccountId(client, dev_token, access_token, user_id) - ## Llamada 3 - - ## Llamada 4 + audiences_list = callGetCustomerListAudiencesByAccounts(client, dev_token, access_token, user_id, account_id) ## Build Up del csv + email = row_param_values["email"] + + csv_file = createCSVForRemovingCustomerListObject(audiences_list, email) + + ## Llamada 4 ## Llamada 5. Enviar el CSv @@ -264,3 +277,26 @@ def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, develope ) return audiences_list + + +def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_email: str): + """ + Createsa CSV with the values to remove the Customer List Objects. + Since we dont know on Which Audience the Customer List Object is, we will remove it from all the Audiences + """ + + base_filepath = "src/fides/api/service/saas_request/upload_files_templates/CustomerListRemoval.csv" + destination = "CustomerListRemoval.csv" + csv_headers = ["Type","Status","Id","Parent Id","Client Id","Modified Time","Name","Description","Scope","Audience","Action Type","SubType","Text"] + + hashedEmail=hashlib.sha256(target_email.encode()).hexdigest() + + shutil.copyfile(base_filepath, destination) + + with open(destination,'a') as csvfile: + writer = csv.DictWriter(csvfile,csv_headers) + for audience_id in audiences_ids: + writer.writerow({"Type": "Customer List", "Id": audience_id, "Client Id": "fides_ethyca", "Action Type": "Update" }) + writer.writerow({"Type": "Customer List Item", "Parent Id": audience_id, "Action Type": "Delete", "SubType": "Email", "Text": hashedEmail}) + + return destination diff --git a/src/playground/fixtures/CustomerListRemoval.csv b/src/fides/api/service/saas_request/upload_files_templates/CustomerListRemoval.csv similarity index 100% rename from src/playground/fixtures/CustomerListRemoval.csv rename to src/fides/api/service/saas_request/upload_files_templates/CustomerListRemoval.csv From 9080d635fe3a9baca9d5e224ed32dcab2432ccb3 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 21 Aug 2024 11:02:15 -0400 Subject: [PATCH 09/39] Creating the CSV and sending the Bulk Upload We have a tracking ID as a response that we do not use yet, We might want to use it on the future --- ...microsoft_advertising_request_overrides.py | 100 ++++++++++++++++-- 1 file changed, 94 insertions(+), 6 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 109ff86518..30e3cd0ab5 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -1,6 +1,7 @@ import hashlib import shutil import csv +import json from typing import Any, Dict, List from defusedxml import ElementTree @@ -75,17 +76,13 @@ def microsoft_advertising_user_delete( audiences_list = callGetCustomerListAudiencesByAccounts(client, dev_token, access_token, user_id, account_id) - ## Build Up del csv email = row_param_values["email"] - csv_file = createCSVForRemovingCustomerListObject(audiences_list, email) - ## Llamada 4 - - ## Llamada 5. Enviar el CSv + upload_url = getBulkUploadURL(client, dev_token, access_token, user_id, account_id) + bulkUploadCustomerList(client, upload_url, csv_file, dev_token, access_token, user_id, account_id) - cliet.send(SaasRequestParams()) rows_updated += 1 return rows_updated @@ -300,3 +297,94 @@ def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_emai writer.writerow({"Type": "Customer List Item", "Parent Id": audience_id, "Action Type": "Delete", "SubType": "Email", "Text": hashedEmail}) return destination + + +def getUploadURLFromResponse(xmlRoot: ElementTree.Element): + for branch in xmlRoot: + if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): + for leaf in branch: + if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetBulkUploadUrlResponse"): + for subleaf in leaf: + if(subleaf.tag== "{https://bingads.microsoft.com/CampaignManagement/v13}UploadUrl"): + return subleaf.text + +def getBulkUploadURL(client: AuthenticatedClient, developer_token: str, authentication_token: str, user_id: str, account_id: str): + + payload = "\n \n GetBulkUploadUrl\n " + authentication_token + "\n " + account_id + "\n " + user_id +"\n "+ developer_token + "\n \n \n \n ErrorsAndResults\n " + account_id + "\n \n \n\n" + headers = { + 'Content-Type': 'text/xml', + 'SOAPAction': 'GetBulkUploadUrl', + } + + client.client_config.host = sandbox_bulk_api_url + + request_params = SaaSRequestParams( + method=HTTPMethod.POST, + path="", + headers=headers, + body=payload + ) + + response = client.send( + request_params + ) + + + context_logger = logger.bind( + **request_details( + client.get_authenticated_request(request_params), response + ) + ) + + upload_url = getUploadURLFromResponse(ElementTree.fromstring(response.text)) + + if not upload_url: + context_logger.error( + "GetBulkUploadUrl collected No upload URL {}.", response.text + ) + + return upload_url + + +### Step 6 : Upload to the API + +def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str ,developer_token: str, authentication_token: str, user_id: str, account_id: str ): + + + payload = { + 'AuthenticationToken': authentication_token, + 'DeveloperToken': developer_token, + 'CustomerId': user_id, + 'AccountId': account_id} + + files=[ + ('uploadFile',('customerlist.csv',open(filepath,'rb'),'application/octet-stream')) + ] + + + ## TODO: Expand SaasRequestParams and AuthenticatedClient to send files + request_params = SaaSRequestParams( + method=HTTPMethod.POST, + path="", + body=payload + files=files + ) + + response = client.send( + request_params + ) + + context_logger = logger.bind( + **request_details( + client.get_authenticated_request(request_params), response + ) + ) + + parsedResponse = json.loads(response.text) + + ## Do we need a process to check the status of the Upload? + context_logger.error( + "Tracking ID of the Upload: {}.", parsedResponse["TrackingId"] + ) + + return True From c215989c7901836fe4f9f62f20aac0ba5af105c1 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 21 Aug 2024 11:06:46 -0400 Subject: [PATCH 10/39] Updating SaaSPreparedRequests to send Files Updated on Authenticated Client. Since its an Optional Parameter, and its on the end, we should not have problems with this change --- src/fides/api/schemas/saas/shared_schemas.py | 6 ++++++ .../api/service/connectors/saas/authenticated_client.py | 1 + .../microsoft_advertising_request_overrides.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/fides/api/schemas/saas/shared_schemas.py b/src/fides/api/schemas/saas/shared_schemas.py index 0dc94402f2..ad5f0f76d9 100644 --- a/src/fides/api/schemas/saas/shared_schemas.py +++ b/src/fides/api/schemas/saas/shared_schemas.py @@ -27,6 +27,12 @@ class SaaSRequestParams(BaseModel): query_params: Dict[str, Any] = {} body: Optional[str] = None model_config = ConfigDict(use_enum_values=True) + files: Optional[list] + + class Config: + """Using enum values""" + + use_enum_values = True class ConnectorParamRef(BaseModel): diff --git a/src/fides/api/service/connectors/saas/authenticated_client.py b/src/fides/api/service/connectors/saas/authenticated_client.py index 6d78dfb232..a230fff684 100644 --- a/src/fides/api/service/connectors/saas/authenticated_client.py +++ b/src/fides/api/service/connectors/saas/authenticated_client.py @@ -72,6 +72,7 @@ def get_authenticated_request( headers=request_params.headers, params=request_params.query_params, data=request_params.body, + files=request_params.files ).prepare() # add authentication if provided diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 30e3cd0ab5..aac4c35c0c 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -366,7 +366,7 @@ def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str request_params = SaaSRequestParams( method=HTTPMethod.POST, path="", - body=payload + body=payload, files=files ) From 8cdaec9fdb3a37b765ffe8d39064fb8d4694cf6b Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 21 Aug 2024 11:07:18 -0400 Subject: [PATCH 11/39] Cleaning Up Bing Script and Playground --- src/playground/bings_script.py | 226 --------------------------------- 1 file changed, 226 deletions(-) delete mode 100644 src/playground/bings_script.py diff --git a/src/playground/bings_script.py b/src/playground/bings_script.py deleted file mode 100644 index 5bedb088c3..0000000000 --- a/src/playground/bings_script.py +++ /dev/null @@ -1,226 +0,0 @@ - -## Placeholder for the Bings Ads Script -### Bing Script - -## Programatically getting the basic for this Bing Ads Call - -## V0.1 - -### Required packages: -import requests -import csv -import hashlib -import shutil -import json -### IMPORTANT: The Base XML package of python is prone to vulnerabilities. I.E Billion Laughs Bomb. Using safer defusedxml package -from defusedxml.ElementTree import fromstring - -### Functions Setup - -## Why do we put the payload on one line: Payloads Fail with line break. - - -## Step 1: Retrieval of User ID - -# Traverses the GetUserRequest SOAP XML tree response and retrieves the User ID -def getUserIdFromResponse(xmlRoot): - ## TODO: Check if we can avoid this Nesting mess - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}GetUserResponse"): - for subleaf in leaf: - if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}User"): - for user_leaf in subleaf: - if(user_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): - return user_leaf.text - -# Container Function of the Get User Process -def callGetUserRequestAndRetrieveUserId(developer_token, authentication_token): - customer_manager_service_url = "https://clientcenter.api.sandbox.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" - payload = "\n \n " + developer_token + "\n " + authentication_token + "\n \n \n \n \n \n \n" - headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'GetUser' - } - - response = requests.request("POST", customer_manager_service_url, headers=headers, data=payload) - - #print( "#####////######") - #print(response.text) - #print( "#####////######") - - return getUserIdFromResponse(fromstring(response.text)) - -## Step 2 : Retrieve the Account -## TODO: What happens if the base account has multiple accounts? - -def getAccountIdFromResponse(xmlRoot): - ## TODO: Check if we can avoid this Nesting mess - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}SearchAccountsResponse"): - for subleaf in leaf: - ## TODO: Expand for Multiple accounts - if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}Accounts"): - for account_leaf in subleaf: - if(account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}AdvertiserAccount"): - for ads_account_leaf in account_leaf: - if(ads_account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): - return ads_account_leaf.text - -def callGetAccountRequestAndRetrieveAccountId(developer_token, authentication_token, user_id): - customer_manager_service_url = "https://clientcenter.api.sandbox.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" - payload = "\n\n \n \n "+ authentication_token + "\n " + developer_token + "\n \n \n \n \n \n UserId\n Equals\n "+ user_id +"\n \n \n \n \n 0\n 10\n \n \n \n\n" - - headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'SearchAccounts' - } - - response = requests.request("POST", customer_manager_service_url, headers=headers, data=payload) - - #print(response.text) - - return getAccountIdFromResponse(fromstring(response.text)) - -### Step 3 - -def getAudiencesIDFromLeaf(xmlLeaf): - ## TODO: Check if we can avoid this Nesting mess - audience_ids = [] - for subleaf in xmlLeaf: - ## TODO: Expand for Multiple accounts - if(subleaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audiences"): - for audience_leaf in subleaf: - if(audience_leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audience"): - for audience_entity in audience_leaf: - if(audience_entity.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Id"): - audience_ids.append(audience_entity.text) - break - - return audience_ids - - -def getAudiencesIDsfromResponse(xmlRoot): - ## TODO: Check if we can avoid this Nesting mess - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetAudiencesByIdsResponse"): - return getAudiencesIDFromLeaf(leaf) - - -def callGetCustomerListAudiencesByAccounts(developer_token, authentication_token, user_id, account_id): - campaing_manager_service_url = "https://campaign.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" - payload = "\n \n GetAudiencesByIds\n " + authentication_token + "\n " + account_id + "\n " + user_id + "\n " + developer_token + "\n \n \n \n CustomerList\n \n \n\n" - headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'GetAudiencesByIds', - } - - response = requests.request("POST", campaing_manager_service_url, headers=headers, data=payload) - return getAudiencesIDsfromResponse(fromstring(response.text)) - -### Step 4 > Build the CSV -def createCSVForRemovingCustomerListObject(audiences_ids,target_email): - base_filepath = "fixtures/CustomerListRemoval.csv" - destination = "CustomerListRemoval.csv" - csv_headers = ["Type","Status","Id","Parent Id","Client Id","Modified Time","Name","Description","Scope","Audience","Action Type","SubType","Text"] - print("Target Email to remove:" + target_email) - ## Hash the Email - hashedEmail=hashlib.sha256(target_email.encode()).hexdigest() - print(" Hashed Email>> " + hashedEmail) - ## Copy the Fixture File - shutil.copyfile(base_filepath, destination) - - with open(destination,'a') as csvfile: - writer = csv.DictWriter(csvfile,csv_headers) - for audience_id in audiences_ids: - writer.writerow({"Type": "Customer List", "Id": audience_id, "Client Id": "fides_ethyca", "Action Type": "Update" }) - writer.writerow({"Type": "Customer List Item", "Parent Id": audience_id, "Action Type": "Delete", "SubType": "Email", "Text": hashedEmail}) - -### Step 5 > Get the Bulk Upload URL - -def getUploadURLFromResponse(xmlRoot): - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetBulkUploadUrlResponse"): - for subleaf in leaf: - if(subleaf.tag== "{https://bingads.microsoft.com/CampaignManagement/v13}UploadUrl"): - return subleaf.text - -def getBulkUploadURL(developer_token, authentication_token, user_id, account_id): - bulk_api_url = "https://bulk.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc?wsdl" - - payload = "\n \n GetBulkUploadUrl\n " + authentication_token + "\n " + account_id + "\n " + user_id +"\n "+ developer_token + "\n \n \n \n ErrorsAndResults\n " + account_id + "\n \n \n\n" - headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'GetBulkUploadUrl', - } - - response = requests.request("POST", bulk_api_url, headers=headers, data=payload) - - return getUploadURLFromResponse(fromstring(response.text)) - -### Step 6 : Upload to the API - -def bulkUploadCustomerList(url,developer_token, authentication_token, user_id, account_id ): - - url = "https://fileupload.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/FileUpload/File/UploadBulkFile/4bba40a0-3f2c-4242-ab08-3c166e833586?Ver=13" - - payload = {'AuthenticationToken': authentication_token, - 'DeveloperToken': developer_token, - 'CustomerId': user_id, - 'AccountId': account_id} - files=[ - ('uploadFile',('customerlist.csv',open('CustomerListRemoval.csv','rb'),'application/octet-stream')) - ] - - headers = { - 'Authorization': 'Bearer' + authentication_token - } - - response = requests.request("POST", url, headers=headers, data=payload, files=files) - - return json.loads(response.text) - - -### Variables Setup -## On the Bings Sandbox enviroment, we dont need to specify the Dev token, since its always the same Token -## Outside Sanbox, we would need to set up the Flow to get the developer token -# Oh what that means we need to do it, dosent it? /s - -sandbox_developer_token = "BBD37VB98" -current_authentication_token = "eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhDQkMtSFMyNTYiLCJ4NXQiOiJJekQ3cTdrcUowdzU0R0wxVUJtT3FHMU9JS0kiLCJ6aXAiOiJERUYifQ.KMko1d7qFCrt65CypzFmuCqt25oHhEhmiie1lZV_FxeP8HzbOjBIW4oJHxwRSQ79SIGn-Ngi6z-32uraZW_cNgBv0UrGWqxOEbhTl6qluDOjP3qiKGu0qGz3tOspjsr6EGXf0118qHXx3Tvo2KrwCPNayUMU7PKsBsm9XuLd2NoTc5-f2gRd-o6-p1-4Zo91mwpRA-FemZpgjFHhH5EYmwgOTwAC9nqLLqtIuuzqjXIkdDLSwxbkw9tINDtrgBjLcijDl3shfWGkpYYgeilP0b56lQUMQq07P7AVzXk8r7KGr_KNUDsRM24MBgCbGJeLRQzCHdu0A2BcpqwwEwkI6A.SfQdsBdwtOC3aBQbh-dLPQ.iTaeopzgt17VAbAKf6H2eyIguyFmqVB3iH2rFTNn-yHHBCh3VJOa4X7NK9bz4NMEZvDZvCoJZz7BquD3H6UPDesk-g_mNF3sS94S1pKg9TgVYQytRJBpjNC-8UmFEpKORK67et2i-GTUYHNzlOsrp5E26WArbeT7RJmxOW7iYh9hrp1Fkk7_87ofi3gHoHmrsaKBQqMnJFnNWHvAzaSz2nExfyl1t5zvg4MxgysgvPsdO5QQIcwlYCTSLfEztqG57cZ1TDOE-QbwwIh8ocCb435ENvi9qXvULcN10K-FMtTXKu96wvPV_fLx2q0B_ohixRIQtronXRfr1ZYGMhsZ02gNnl3IZuDcAobUAv_QfxJJgC7Je311ouMv9zC23-hI7poPYHUgHxWvziaz_vmRqMu2X90bEGU7XL5QECUM8W1uPl1jYzlGYZsjMj2gxkJ8UvIx3euOW3l1_ZTV5xShh9UxBrx66mwYvApO8WmGc8IYeCg7KmgUvVZmIF03rW5feK8PCzt84HsAMqU9shCcLWmhrByoB53ew7LuzjPIEB5GG-ZphqLdDSMp4Wr_u0_-mjZWxVK0Vtk2kCK6WgXjT0fgpkAxj5kH4SvtzF5L6l3Gu9B-VtF2ywoy9Ec4065p-Fi3vAoaAs6RPWMlp0Oc05PErawv1P1myotAb9QocRbhHQwD2LmX5_1vlIQ2lfxh6TXYk64NMqr3bYsMiG2M7PXC9JpgH-FbgM7LVgUDKYxArBPg3gi5vGbbSL7gyVbhAJgDwZzyYHLkUzhavS2NpUTyxdrm0cPzvXa9R130Q3AjXplbm4DZv03Y9r9sO8IM0HYMZBzkL4RTKV98DUIIe89_SOdjA8pZk1DG86KleOjyk9-CXJTWiZZP8wurNrtZ3Z4bryaagQbg-AkI4AjsrGrwcTFpWIdLXm4p98t0ByYcUdnrH3z31O45CWeQXacQOQk1tD6ixqWS1Hikkv8CoAfxwQvRSWNiM2UmNnxp28i2t30cEg2e2wN9V3xSDaftwcE1dyrKKryr4Ql37Wbv7GfSE44MAFyoNaJpjUx59L98dbPGfTLOM81gwC2arW8DYsSHmsctYTea-EwSKpwGRbbQyLf1LDd0YgADAk63X9GMyIBhehQp9Z99jglt1HStmiAsNpQno276_ndCdJzEtlnErYNCZ1JaUjffzPn7R1rztltadjlOCppC0VdqwxyAHNiH0EL0JSeQcKzbfUUzeC_5iYUJrAMcDsSwB9cqh2QX2v4gljKeN1gWUGGzWWQ3rxj2SH8abJHFdqB1tT9OxFSeqFbTDZxp_qces0Xs-2Vnj2xfRofab88hyVQ_tgN-XsXY9PservIoDA1_RhLytiMdmy6e0HevBWr421X_PktzHTZW5IR3XiZOBcNLEa9kEGkZTkx8bWnrVJlyLTqUemo6yPZzJ5hdCvDZvoJrcz807mibp3MywjVke2xyt0xDVnV7hw7WAEor4uEekE_6PA.YLzROApiy8XWW9BKmGwLYg" - -print("Call numero One ######") -user_id = callGetUserRequestAndRetrieveUserId(sandbox_developer_token, current_authentication_token) -print("Gathered User Id:",user_id) - -print("Call Numbe Dos ######") -account_id = callGetAccountRequestAndRetrieveAccountId(sandbox_developer_token, current_authentication_token, user_id) -print("Gathered Account Id:",account_id) - -print ("Llamada Number 3") -audience_ids = callGetCustomerListAudiencesByAccounts(sandbox_developer_token,current_authentication_token, user_id, account_id) -print("Gathered Audience IDs:") -print(audience_ids) - -print("Building the CSV") -createCSVForRemovingCustomerListObject(audience_ids,"someemail@data.cl") - -print("Llamada 4, Obteniendo el Bulk Upload URL") - -upload_url = getBulkUploadURL(sandbox_developer_token,current_authentication_token,user_id,account_id) - -print("Upload URL: "+ upload_url) - -print("Llamada 5: Uploading Customer List File") -response = bulkUploadCustomerList(upload_url,sandbox_developer_token,current_authentication_token,user_id,account_id) -print(response["TrackingId"]) - - - From 80874ed85dc614276d31a7b2b058b85ba2d527a8 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 21 Aug 2024 17:24:55 -0400 Subject: [PATCH 12/39] implementing XPath to avoid nesting loops --- ...microsoft_advertising_request_overrides.py | 104 +++++++++--------- 1 file changed, 50 insertions(+), 54 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index aac4c35c0c..d74a1b29dc 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -10,7 +10,7 @@ from fides.api.models.policy import Policy from fides.api.models.privacy_request import PrivacyRequest from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient +from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient, RequestFailureResponseException from fides.api.service.saas_request.saas_request_override_factory import ( SaaSRequestType, register, @@ -22,6 +22,13 @@ sandbox_campaing_manager_service_url = "https://campaign.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" sandbox_bulk_api_url = "https://bulk.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc?wsdl" +namespaces = { + 'soap': 'http://schemas.xmlsoap.org/soap/envelope/', + 'ms_customer': 'https://bingads.microsoft.com/Customer/v13', + 'ms_campaign': 'https://bingads.microsoft.com/CampaignManagement/v13', + + 'ent': 'https://bingads.microsoft.com/Customer/v13/Entities' +} @register("microsoft_advertising_test_connection", [SaaSRequestType.TEST]) def microsoft_advertising_test_connection( @@ -91,16 +98,15 @@ def getUserIdFromResponse(xmlRoot: ElementTree.Element): """ Retrieves the ID from the expected XML response of the GetUserRequest """ - ## TODO: Check if we can avoid this Nesting mess - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}GetUserResponse"): - for subleaf in leaf: - if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}User"): - for user_leaf in subleaf: - if(user_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): - return user_leaf.text + xpath = './soap:Body/ms_customer:GetUserResponse/ms_customer:User/ent:Id' + id_element = xmlRoot.find(xpath, namespaces) + + + if id_element is not None: + return id_element.text + else: + return None + # or raise an exception, depending on your error handling strategy def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_token: str, authentication_token: str): @@ -140,26 +146,26 @@ def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_ "GetUser request failed with the following message {}.", response.text ) + raise RequestFailureResponseException(response=response) + + return user_id + def getAccountIdFromResponse(xmlRoot: ElementTree.Element): """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts """ - ## TODO: Check if we can avoid this Nesting mess - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/Customer/v13}SearchAccountsResponse"): - for subleaf in leaf: - ## TODO: Expand for Multiple accounts Here - if(subleaf.tag== "{https://bingads.microsoft.com/Customer/v13}Accounts"): - for account_leaf in subleaf: - if(account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}AdvertiserAccount"): - for ads_account_leaf in account_leaf: - if(ads_account_leaf.tag == "{https://bingads.microsoft.com/Customer/v13/Entities}Id"): - return ads_account_leaf.text + # Use XPath to directly find the Id element + xpath = './soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts/ent:AdvertiserAccount/ent:Id' + id_element = xmlRoot.find(xpath, namespaces) + + if id_element is not None: + return id_element.text + else: + return None # or raise an exception, depending on your error handling strategy + def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , developer_token: str, authentication_token: str, user_id: str): """ @@ -202,35 +208,26 @@ def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , deve return accountId -def getAudiencesIDFromLeaf(xmlLeaf: ElementTree.Element): +def getAudiencesIDsfromResponse(xmlRoot: ElementTree.Element): + """ - Gets the Audiences from the XML Node extracted from the GetAudiencesByIdsResponse + Gets the Audience _ids from the GetAudiencesByIdsResponse """ - ## TODO: Check if we can avoid this Nesting mess + audience_ids = [] - for subleaf in xmlLeaf: - ## TODO: Expand for Multiple accounts having the same Audiences - if(subleaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audiences"): - for audience_leaf in subleaf: - if(audience_leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Audience"): - for audience_entity in audience_leaf: - if(audience_entity.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}Id"): - audience_ids.append(audience_entity.text) - break + xpath = './soap:Body/ms_campaign:GetAudiencesByIdsResponse/ms_campaign:Audiences' + audiences_element = xmlRoot.find(xpath, namespaces) - return audience_ids + + for audience_leaf in audiences_element: + xmlSubpath = './ms_campaign:Id' + print(audience_leaf) + audience_id = audience_leaf.find(xmlSubpath, namespaces) + if audience_id is not None: + audience_ids.append(audience_id.text) + return audience_ids -def getAudiencesIDsfromResponse(xmlRoot:ElementTree.Element): - """ - Gets the Audience Leaf nodes from the GetAudiencesByIdsResponse - """ - ## TODO: Check if we can avoid this Nesting mess - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetAudiencesByIdsResponse"): - return getAudiencesIDFromLeaf(leaf) def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, developer_token:str, authentication_token:str , user_id:str, account_id: str): @@ -300,13 +297,12 @@ def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_emai def getUploadURLFromResponse(xmlRoot: ElementTree.Element): - for branch in xmlRoot: - if(branch.tag == "{http://schemas.xmlsoap.org/soap/envelope/}Body"): - for leaf in branch: - if(leaf.tag == "{https://bingads.microsoft.com/CampaignManagement/v13}GetBulkUploadUrlResponse"): - for subleaf in leaf: - if(subleaf.tag== "{https://bingads.microsoft.com/CampaignManagement/v13}UploadUrl"): - return subleaf.text + + xpath = './soap:Body/ms_campaign:GetBulkUploadUrlResponse/ms_campaign:UploadUrl' + upload_url_element = xmlRoot.find(xpath, namespaces) + + return upload_url_element.text + def getBulkUploadURL(client: AuthenticatedClient, developer_token: str, authentication_token: str, user_id: str, account_id: str): From 23efb451da3d49d4fe2fbc5dd1848d6a64ca4901 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 21 Aug 2024 17:27:12 -0400 Subject: [PATCH 13/39] preparing Productiond endpoints in config --- data/saas/config/microsoft_advertising_config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/saas/config/microsoft_advertising_config.yml b/data/saas/config/microsoft_advertising_config.yml index cc19828336..5c2d3e9d4d 100644 --- a/data/saas/config/microsoft_advertising_config.yml +++ b/data/saas/config/microsoft_advertising_config.yml @@ -32,6 +32,7 @@ saas_config: client_config: protocol: https host: login.windows-ppe.net + ##host: login.microsoftonline.com path: /consumers/oauth2/v2.0/authorize query_params: - name: client_id @@ -49,6 +50,7 @@ saas_config: client_config: protocol: https host: login.windows-ppe.net + ##host: login.microsoftonline.com path: /consumers/oauth2/v2.0/token headers: - name: Content-Type @@ -66,6 +68,7 @@ saas_config: client_config: protocol: https host: login.windows-ppe.net + ##host: login.microsoftonline.com path: /consumers/oauth2/v2.0/token headers: - name: Content-Type From e2c68ca83b1bb16bc9003d3f293e608ede021f8b Mon Sep 17 00:00:00 2001 From: bruno Date: Thu, 22 Aug 2024 11:18:34 -0400 Subject: [PATCH 14/39] Managing Common Requests Failures if they dont comply to the expect format, we wont get a result from the relevant fields. Thus there was an error with the response. response.text should yield the XML string containing the Error description from the endpoint --- ...microsoft_advertising_request_overrides.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index d74a1b29dc..cd075b24c8 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -32,6 +32,7 @@ @register("microsoft_advertising_test_connection", [SaaSRequestType.TEST]) def microsoft_advertising_test_connection( + client: AuthenticatedClient, param_values_per_row: List[Dict[str, Any]], policy: Policy, @@ -39,18 +40,20 @@ def microsoft_advertising_test_connection( secrets: Dict[str, Any], is_sandbox: bool = False ) -> int: + """ + Tests the Microsoft Advertising Connection + + Attempts to retrieve the User ID from the Microsoft Advertising API, checking that the tokens are valid tokens + + """ rows_updated = 0 access_token = secrets["access_token"] dev_token = secrets["dev_token"] - for row_param_values in param_values_per_row: - # API calls go here, look at other request overrides in this directory as examples + callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) - - rows_updated += 1 - - return rows_updated + return rows_updated+1 @@ -205,6 +208,9 @@ def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , deve "SearchAccounts request failed with the following message {}.", response.text ) + raise RequestFailureResponseException(response=response) + + return accountId @@ -269,6 +275,8 @@ def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, develope context_logger.error( "GetAudiencesByIds collected No audiences {}.", response.text ) + raise RequestFailureResponseException(response=response) + return audiences_list @@ -338,6 +346,8 @@ def getBulkUploadURL(client: AuthenticatedClient, developer_token: str, authenti context_logger.error( "GetBulkUploadUrl collected No upload URL {}.", response.text ) + raise RequestFailureResponseException(response=response) + return upload_url @@ -378,9 +388,16 @@ def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str parsedResponse = json.loads(response.text) + if not parsedResponse["TrackingId"]: + context_logger.error( + "GetBulkUploadUrl collected No upload URL {}.", response.text + ) + raise RequestFailureResponseException(response=response) + ## Do we need a process to check the status of the Upload? - context_logger.error( - "Tracking ID of the Upload: {}.", parsedResponse["TrackingId"] + ## How are we persisting data? + context_logger.info( + "Tracking ID of the recent upload: {}.", parsedResponse["TrackingId"] ) return True From 162b8a1e8ca6293fb0ff2d9db40ca16548ff295f Mon Sep 17 00:00:00 2001 From: bruno Date: Thu, 22 Aug 2024 17:23:38 -0400 Subject: [PATCH 15/39] Adding Base Fixtures Data --- tests/fixtures/saas/microsoft_advertising_fixtures.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/saas/microsoft_advertising_fixtures.py b/tests/fixtures/saas/microsoft_advertising_fixtures.py index 18e1520395..47bb85909e 100644 --- a/tests/fixtures/saas/microsoft_advertising_fixtures.py +++ b/tests/fixtures/saas/microsoft_advertising_fixtures.py @@ -15,8 +15,11 @@ @pytest.fixture(scope="session") def microsoft_advertising_secrets(saas_config) -> Dict[str, Any]: return { - "domain": pydash.get(saas_config, "microsoft_advertising.domain") - or secrets["domain"] + "domain": pydash.get(saas_config, "microsoft_advertising.domain") or secrets["domain"] , + "client_id" :pydash.get(saas_config, "microsoft_advertising.client_id") or secrets["client_id"] , + "client_secret" :pydash.get(saas_config, "microsoft_advertising.secret_id") or secrets["secret_id"] , + "dev_token": pydash.get(saas_config, "microsoft_advertising.dev_token") or secrets["dev_token"] , + 'access_token': pydash.get(saas_config, "microsoft_advertising.access_token") or secrets["access_token"] # add the rest of your secrets here } From a4a2199684ea4caea4b0e68256d3f116f1b96cc3 Mon Sep 17 00:00:00 2001 From: bruno Date: Thu, 22 Aug 2024 17:23:48 -0400 Subject: [PATCH 16/39] Adding Endpoints for Production --- .../microsoft_advertising_request_overrides.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index cd075b24c8..6868e7c67d 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -22,6 +22,10 @@ sandbox_campaing_manager_service_url = "https://campaign.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" sandbox_bulk_api_url = "https://bulk.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc?wsdl" +customer_manager_service_url = "https://clientcenter.api.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" +campaing_manager_service_url = "https://campaign.api.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" +bulk_api_url = "https://bulk.api.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc" + namespaces = { 'soap': 'http://schemas.xmlsoap.org/soap/envelope/', 'ms_customer': 'https://bingads.microsoft.com/Customer/v13', @@ -97,7 +101,7 @@ def microsoft_advertising_user_delete( return rows_updated -def getUserIdFromResponse(xmlRoot: ElementTree.Element): +def getUserIdFromResponse(xmlRoot: ElementTree): """ Retrieves the ID from the expected XML response of the GetUserRequest """ @@ -224,10 +228,11 @@ def getAudiencesIDsfromResponse(xmlRoot: ElementTree.Element): xpath = './soap:Body/ms_campaign:GetAudiencesByIdsResponse/ms_campaign:Audiences' audiences_element = xmlRoot.find(xpath, namespaces) + if(audiences_element is None): + return None for audience_leaf in audiences_element: xmlSubpath = './ms_campaign:Id' - print(audience_leaf) audience_id = audience_leaf.find(xmlSubpath, namespaces) if audience_id is not None: audience_ids.append(audience_id.text) From 7e16d111bc85948d40f3c5d63ba632db9fb30838 Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 23 Aug 2024 16:41:21 -0400 Subject: [PATCH 17/39] Setting up missing mixture data --- tests/fixtures/saas/microsoft_advertising_fixtures.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/saas/microsoft_advertising_fixtures.py b/tests/fixtures/saas/microsoft_advertising_fixtures.py index 47bb85909e..0c909984f2 100644 --- a/tests/fixtures/saas/microsoft_advertising_fixtures.py +++ b/tests/fixtures/saas/microsoft_advertising_fixtures.py @@ -17,9 +17,10 @@ def microsoft_advertising_secrets(saas_config) -> Dict[str, Any]: return { "domain": pydash.get(saas_config, "microsoft_advertising.domain") or secrets["domain"] , "client_id" :pydash.get(saas_config, "microsoft_advertising.client_id") or secrets["client_id"] , - "client_secret" :pydash.get(saas_config, "microsoft_advertising.secret_id") or secrets["secret_id"] , + "client_secret" :pydash.get(saas_config, "microsoft_advertising.client_secret") or secrets["client_secret"] , "dev_token": pydash.get(saas_config, "microsoft_advertising.dev_token") or secrets["dev_token"] , - 'access_token': pydash.get(saas_config, "microsoft_advertising.access_token") or secrets["access_token"] + 'access_token': pydash.get(saas_config, "microsoft_advertising.access_token") or secrets["access_token"] , + 'redirect_uri': pydash.get(saas_config, "microsoft_advertising.redirect_uri") or secrets["redirect_uri"] # add the rest of your secrets here } From 1b070c3c12c9796c7df80ac7734ecbe1387fe302 Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 23 Aug 2024 18:03:12 -0400 Subject: [PATCH 18/39] Using Client URI for proper targeting of endpoints --- ...microsoft_advertising_request_overrides.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 6868e7c67d..c342a7ad5a 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -36,7 +36,6 @@ @register("microsoft_advertising_test_connection", [SaaSRequestType.TEST]) def microsoft_advertising_test_connection( - client: AuthenticatedClient, param_values_per_row: List[Dict[str, Any]], policy: Policy, @@ -48,7 +47,7 @@ def microsoft_advertising_test_connection( Tests the Microsoft Advertising Connection Attempts to retrieve the User ID from the Microsoft Advertising API, checking that the tokens are valid tokens - + """ rows_updated = 0 @@ -101,7 +100,10 @@ def microsoft_advertising_user_delete( return rows_updated -def getUserIdFromResponse(xmlRoot: ElementTree): +def getUserIdFromResponse(xmlRoot): + + print(type(xmlRoot)) + """ Retrieves the ID from the expected XML response of the GetUserRequest """ @@ -123,7 +125,7 @@ def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_ payload = "\n \n " + developer_token + "\n " + authentication_token + "\n \n \n \n \n \n \n" - client.client_config.host = sandbox_customer_manager_service_url + client.uri = customer_manager_service_url headers = { 'Content-Type': 'text/xml', @@ -159,7 +161,7 @@ def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_ return user_id -def getAccountIdFromResponse(xmlRoot: ElementTree.Element): +def getAccountIdFromResponse(xmlRoot ): """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts @@ -167,12 +169,12 @@ def getAccountIdFromResponse(xmlRoot: ElementTree.Element): # Use XPath to directly find the Id element xpath = './soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts/ent:AdvertiserAccount/ent:Id' id_element = xmlRoot.find(xpath, namespaces) - + if id_element is not None: return id_element.text else: return None # or raise an exception, depending on your error handling strategy - + def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , developer_token: str, authentication_token: str, user_id: str): """ @@ -186,7 +188,7 @@ def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , deve 'SOAPAction': 'SearchAccounts' } - client.client_config.host = sandbox_customer_manager_service_url + client.uri = customer_manager_service_url request_params = SaaSRequestParams( method=HTTPMethod.POST, @@ -218,8 +220,8 @@ def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , deve return accountId -def getAudiencesIDsfromResponse(xmlRoot: ElementTree.Element): - +def getAudiencesIDsfromResponse(xmlRoot ): + """ Gets the Audience _ids from the GetAudiencesByIdsResponse """ @@ -230,7 +232,7 @@ def getAudiencesIDsfromResponse(xmlRoot: ElementTree.Element): if(audiences_element is None): return None - + for audience_leaf in audiences_element: xmlSubpath = './ms_campaign:Id' audience_id = audience_leaf.find(xmlSubpath, namespaces) @@ -249,7 +251,7 @@ def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, develope 'SOAPAction': 'GetAudiencesByIds', } - client.client_config.host = sandbox_campaing_manager_service_url + client.uri = campaing_manager_service_url request_params = SaaSRequestParams( method=HTTPMethod.POST, @@ -309,7 +311,7 @@ def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_emai return destination -def getUploadURLFromResponse(xmlRoot: ElementTree.Element): +def getUploadURLFromResponse(xmlRoot ): xpath = './soap:Body/ms_campaign:GetBulkUploadUrlResponse/ms_campaign:UploadUrl' upload_url_element = xmlRoot.find(xpath, namespaces) @@ -325,7 +327,7 @@ def getBulkUploadURL(client: AuthenticatedClient, developer_token: str, authenti 'SOAPAction': 'GetBulkUploadUrl', } - client.client_config.host = sandbox_bulk_api_url + client.uri = bulk_api_url request_params = SaaSRequestParams( method=HTTPMethod.POST, From e7172fdc3f73617ad447281ca4299c7ff2d6388b Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 23 Aug 2024 18:05:12 -0400 Subject: [PATCH 19/39] Adding required parameter on the delete request --- .../integration_tests/saas/test_microsoft_advertising_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py index 30fd53f363..fe87cb7ac6 100644 --- a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py +++ b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py @@ -6,7 +6,8 @@ @pytest.mark.integration_saas class TestMicrosoftAdvertisingConnector: - def test_connection(self, microsoft_advertising_runner: ConnectorRunner): + def test_connection( + self, microsoft_advertising_runner: ConnectorRunner): microsoft_advertising_runner.test_connection() async def test_non_strict_erasure_request( @@ -15,7 +16,6 @@ async def test_non_strict_erasure_request( policy: Policy, erasure_policy_string_rewrite: Policy, microsoft_advertising_erasure_identity_email: str, - microsoft_advertising_erasure_data, ): ( _, From 5b22de6c7b96070b0132e659df9225b91033d535 Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 23 Aug 2024 18:09:51 -0400 Subject: [PATCH 20/39] Using request override for the test connection Still with some problems in local. The configuration is not being sent properly --- data/saas/config/microsoft_advertising_config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data/saas/config/microsoft_advertising_config.yml b/data/saas/config/microsoft_advertising_config.yml index 5c2d3e9d4d..4ef3045070 100644 --- a/data/saas/config/microsoft_advertising_config.yml +++ b/data/saas/config/microsoft_advertising_config.yml @@ -83,8 +83,7 @@ saas_config: } test_request: - method: GET - path: /api/rest/v6/users # update this with a simple API call that uses the token, just to verify + request_override: microsoft_advertising_test_connection endpoints: - name: user From a2284f91d1b97aafeab31f2adf66e7601a68ca91 Mon Sep 17 00:00:00 2001 From: bruno Date: Fri, 23 Aug 2024 18:28:42 -0400 Subject: [PATCH 21/39] Setting up for multiple audiences --- ...microsoft_advertising_request_overrides.py | 53 ++++++++++++------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index c342a7ad5a..62d33cfa59 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -85,18 +85,23 @@ def microsoft_advertising_user_delete( user_id = callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) - account_id = callGetAccountRequestAndRetrieveAccountId(client, dev_token, access_token, user_id) + account_ids = callGetAccountRequestAndRetrieveAccountsId(client, dev_token, access_token, user_id) - audiences_list = callGetCustomerListAudiencesByAccounts(client, dev_token, access_token, user_id, account_id) + for account_id in account_ids: - email = row_param_values["email"] - csv_file = createCSVForRemovingCustomerListObject(audiences_list, email) + audiences_list = callGetCustomerListAudiencesByAccounts(client, dev_token, access_token, user_id, account_id) - upload_url = getBulkUploadURL(client, dev_token, access_token, user_id, account_id) + if audiences_list is None: + continue - bulkUploadCustomerList(client, upload_url, csv_file, dev_token, access_token, user_id, account_id) + email = row_param_values["email"] + csv_file = createCSVForRemovingCustomerListObject(audiences_list, email) - rows_updated += 1 + upload_url = getBulkUploadURL(client, dev_token, access_token, user_id, account_id) + + bulkUploadCustomerList(client, upload_url, csv_file, dev_token, access_token, user_id, account_id) + + rows_updated += 1 return rows_updated @@ -161,22 +166,33 @@ def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_ return user_id -def getAccountIdFromResponse(xmlRoot ): +def getAccountsIdFromResponse(xmlRoot ): """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts """ # Use XPath to directly find the Id element - xpath = './soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts/ent:AdvertiserAccount/ent:Id' + xpath = './soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts/' id_element = xmlRoot.find(xpath, namespaces) - if id_element is not None: - return id_element.text - else: - return None # or raise an exception, depending on your error handling strategy + accounts_id = [] + xpath = './soap:Body/ms_campaign:GetAudiencesByIdsResponse/ms_campaign:Audiences' + accounts_element = xmlRoot.find(xpath, namespaces) + + if(accounts_element is None): + return None + + for account_element in accounts_element: + xmlSubpath = './ent:AdvertiserAccount/ent:Id' + account_id = account_element.find(xmlSubpath, namespaces) + if account_id is not None: + account_element.append(account_id.text) + + return accounts_id + -def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , developer_token: str, authentication_token: str, user_id: str): +def callGetAccountRequestAndRetrieveAccountsId(client: AuthenticatedClient , developer_token: str, authentication_token: str, user_id: str): """ Calls the SearchAccounts SOAP endpoint and retrieves the Account ID from the response """ @@ -207,9 +223,9 @@ def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , deve ) ) - accountId = getAccountIdFromResponse(ElementTree.fromstring(response.text)) + accountIds = getAccountsIdFromResponse(ElementTree.fromstring(response.text)) - if accountId is None: + if accountIds is None: context_logger.error( "SearchAccounts request failed with the following message {}.", response.text ) @@ -217,7 +233,7 @@ def callGetAccountRequestAndRetrieveAccountId(client: AuthenticatedClient , deve raise RequestFailureResponseException(response=response) - return accountId + return accountIds def getAudiencesIDsfromResponse(xmlRoot ): @@ -282,7 +298,8 @@ def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, develope context_logger.error( "GetAudiencesByIds collected No audiences {}.", response.text ) - raise RequestFailureResponseException(response=response) + + return None return audiences_list From f6e5cb23364d690d18df9266675abca06a9a8212 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 10:31:47 -0400 Subject: [PATCH 22/39] Updating Request Overrides for multiple accounts sucessfully We might want to do a deeper Error handling, as there acan be detailed errors on the SOAP Structure --- .../microsoft_advertising_request_overrides.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 62d33cfa59..6c0976e6bd 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -172,21 +172,18 @@ def getAccountsIdFromResponse(xmlRoot ): TODO: Expand for Multiple accounts """ # Use XPath to directly find the Id element - xpath = './soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts/' - id_element = xmlRoot.find(xpath, namespaces) - accounts_id = [] - xpath = './soap:Body/ms_campaign:GetAudiencesByIdsResponse/ms_campaign:Audiences' + xpath = './soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts' accounts_element = xmlRoot.find(xpath, namespaces) if(accounts_element is None): return None - + + xmlSubpath = './ent:Id' for account_element in accounts_element: - xmlSubpath = './ent:AdvertiserAccount/ent:Id' account_id = account_element.find(xmlSubpath, namespaces) if account_id is not None: - account_element.append(account_id.text) + accounts_id.append(account_id.text) return accounts_id @@ -232,6 +229,9 @@ def callGetAccountRequestAndRetrieveAccountsId(client: AuthenticatedClient , dev raise RequestFailureResponseException(response=response) + context_logger.info( + "SearchAccounts request was succesfull with the following ids {}.", accountIds + ) return accountIds @@ -287,10 +287,6 @@ def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, develope ) - response = client.send( - request_params - ) - audiences_list = getAudiencesIDsfromResponse(ElementTree.fromstring(response.text)) if not audiences_list: From 7cdb936be0e91487cb27b0fa86ac68ff9f30b4d0 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 12:07:11 -0400 Subject: [PATCH 23/39] Properly setting up the headers authentication on Bulk Upload Updating Test, using the real reference --- .../microsoft_advertising_request_overrides.py | 10 +++++----- .../saas/test_microsoft_advertising_task.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 6c0976e6bd..948dd0f7f2 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -377,7 +377,7 @@ def getBulkUploadURL(client: AuthenticatedClient, developer_token: str, authenti def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str ,developer_token: str, authentication_token: str, user_id: str, account_id: str ): - payload = { + headers = { 'AuthenticationToken': authentication_token, 'DeveloperToken': developer_token, 'CustomerId': user_id, @@ -387,17 +387,17 @@ def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str ('uploadFile',('customerlist.csv',open(filepath,'rb'),'application/octet-stream')) ] + client.uri = url - ## TODO: Expand SaasRequestParams and AuthenticatedClient to send files request_params = SaaSRequestParams( method=HTTPMethod.POST, + headers = headers, path="", - body=payload, files=files ) - + response = client.send( - request_params + request_params, ) context_logger = logger.bind( diff --git a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py index fe87cb7ac6..e4f58a7d41 100644 --- a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py +++ b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py @@ -25,4 +25,4 @@ async def test_non_strict_erasure_request( erasure_policy=erasure_policy_string_rewrite, identities={"email": microsoft_advertising_erasure_identity_email}, ) - assert erasure_results == {"microsoft_advertising_instance": 1} + assert erasure_results == {"microsoft_advertising_instance:user": 1} From c81eeced25676308279936e6aa3ee0d408916a18 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 14:01:37 -0400 Subject: [PATCH 24/39] Setting Correctly the base variables for the Test Connection override request --- .../microsoft_advertising_request_overrides.py | 8 ++------ .../saas/test_microsoft_advertising_task.py | 7 ++++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 948dd0f7f2..3a793e621e 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -37,11 +37,7 @@ @register("microsoft_advertising_test_connection", [SaaSRequestType.TEST]) def microsoft_advertising_test_connection( client: AuthenticatedClient, - param_values_per_row: List[Dict[str, Any]], - policy: Policy, - privacy_request: PrivacyRequest, secrets: Dict[str, Any], - is_sandbox: bool = False ) -> int: """ Tests the Microsoft Advertising Connection @@ -178,7 +174,7 @@ def getAccountsIdFromResponse(xmlRoot ): if(accounts_element is None): return None - + xmlSubpath = './ent:Id' for account_element in accounts_element: account_id = account_element.find(xmlSubpath, namespaces) @@ -395,7 +391,7 @@ def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str path="", files=files ) - + response = client.send( request_params, ) diff --git a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py index e4f58a7d41..82b507c1c2 100644 --- a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py +++ b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py @@ -7,7 +7,12 @@ @pytest.mark.integration_saas class TestMicrosoftAdvertisingConnector: def test_connection( - self, microsoft_advertising_runner: ConnectorRunner): + self, + microsoft_advertising_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + microsoft_advertising_erasure_identity_email: str + ): microsoft_advertising_runner.test_connection() async def test_non_strict_erasure_request( From 4a1363c8db17316b752b100b583bd19143883b00 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 14:09:00 -0400 Subject: [PATCH 25/39] Using live endpoints in microsoft ads config --- data/saas/config/microsoft_advertising_config.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/data/saas/config/microsoft_advertising_config.yml b/data/saas/config/microsoft_advertising_config.yml index 4ef3045070..2aa839640f 100644 --- a/data/saas/config/microsoft_advertising_config.yml +++ b/data/saas/config/microsoft_advertising_config.yml @@ -27,12 +27,12 @@ saas_config: authentication: strategy: oauth2_authorization_code configuration: + expires_in: 3600 authorization_request: method: GET client_config: protocol: https - host: login.windows-ppe.net - ##host: login.microsoftonline.com + host: login.microsoftonline.com path: /consumers/oauth2/v2.0/authorize query_params: - name: client_id @@ -49,8 +49,7 @@ saas_config: method: POST client_config: protocol: https - host: login.windows-ppe.net - ##host: login.microsoftonline.com + host: login.microsoftonline.com path: /consumers/oauth2/v2.0/token headers: - name: Content-Type @@ -67,8 +66,7 @@ saas_config: method: POST client_config: protocol: https - host: login.windows-ppe.net - ##host: login.microsoftonline.com + host: login.microsoftonline.com path: /consumers/oauth2/v2.0/token headers: - name: Content-Type From 5dcd8cb2731bd46ef4fbef7188ac68e85bb2c976 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 14:27:49 -0400 Subject: [PATCH 26/39] running linters and format checkers --- .../connectors/saas/authenticated_client.py | 2 +- ...microsoft_advertising_request_overrides.py | 311 +++++++++++------- .../saas/microsoft_advertising_fixtures.py | 18 +- .../saas/test_microsoft_advertising_task.py | 2 +- 4 files changed, 202 insertions(+), 131 deletions(-) diff --git a/src/fides/api/service/connectors/saas/authenticated_client.py b/src/fides/api/service/connectors/saas/authenticated_client.py index a230fff684..112feba593 100644 --- a/src/fides/api/service/connectors/saas/authenticated_client.py +++ b/src/fides/api/service/connectors/saas/authenticated_client.py @@ -72,7 +72,7 @@ def get_authenticated_request( headers=request_params.headers, params=request_params.query_params, data=request_params.body, - files=request_params.files + files=request_params.files, ).prepare() # add authentication if provided diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 3a793e621e..fded2a344a 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -1,23 +1,25 @@ -import hashlib -import shutil import csv +import hashlib import json - +import shutil from typing import Any, Dict, List + from defusedxml import ElementTree from loguru import logger from fides.api.models.policy import Policy from fides.api.models.privacy_request import PrivacyRequest from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient, RequestFailureResponseException +from fides.api.service.connectors.saas.authenticated_client import ( + AuthenticatedClient, + RequestFailureResponseException, +) from fides.api.service.saas_request.saas_request_override_factory import ( SaaSRequestType, register, ) from fides.api.util.logger_context_utils import request_details - sandbox_customer_manager_service_url = "https://clientcenter.api.sandbox.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" sandbox_campaing_manager_service_url = "https://campaign.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" sandbox_bulk_api_url = "https://bulk.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc?wsdl" @@ -27,13 +29,13 @@ bulk_api_url = "https://bulk.api.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc" namespaces = { - 'soap': 'http://schemas.xmlsoap.org/soap/envelope/', - 'ms_customer': 'https://bingads.microsoft.com/Customer/v13', - 'ms_campaign': 'https://bingads.microsoft.com/CampaignManagement/v13', - - 'ent': 'https://bingads.microsoft.com/Customer/v13/Entities' + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "ms_customer": "https://bingads.microsoft.com/Customer/v13", + "ms_campaign": "https://bingads.microsoft.com/CampaignManagement/v13", + "ent": "https://bingads.microsoft.com/Customer/v13/Entities", } + @register("microsoft_advertising_test_connection", [SaaSRequestType.TEST]) def microsoft_advertising_test_connection( client: AuthenticatedClient, @@ -52,8 +54,7 @@ def microsoft_advertising_test_connection( callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) - return rows_updated+1 - + return rows_updated + 1 @register("microsoft_advertising_user_delete", [SaaSRequestType.DELETE]) @@ -63,7 +64,7 @@ def microsoft_advertising_user_delete( policy: Policy, privacy_request: PrivacyRequest, secrets: Dict[str, Any], - is_sandbox: bool = False + is_sandbox: bool = False, ) -> int: """ Process of removing an User email from the Microsoft Advertising Platform @@ -81,11 +82,15 @@ def microsoft_advertising_user_delete( user_id = callGetUserRequestAndRetrieveUserId(client, dev_token, access_token) - account_ids = callGetAccountRequestAndRetrieveAccountsId(client, dev_token, access_token, user_id) + account_ids = callGetAccountRequestAndRetrieveAccountsId( + client, dev_token, access_token, user_id + ) for account_id in account_ids: - audiences_list = callGetCustomerListAudiencesByAccounts(client, dev_token, access_token, user_id, account_id) + audiences_list = callGetCustomerListAudiencesByAccounts( + client, dev_token, access_token, user_id, account_id + ) if audiences_list is None: continue @@ -93,14 +98,25 @@ def microsoft_advertising_user_delete( email = row_param_values["email"] csv_file = createCSVForRemovingCustomerListObject(audiences_list, email) - upload_url = getBulkUploadURL(client, dev_token, access_token, user_id, account_id) + upload_url = getBulkUploadURL( + client, dev_token, access_token, user_id, account_id + ) - bulkUploadCustomerList(client, upload_url, csv_file, dev_token, access_token, user_id, account_id) + bulkUploadCustomerList( + client, + upload_url, + csv_file, + dev_token, + access_token, + user_id, + account_id, + ) rows_updated += 1 return rows_updated + def getUserIdFromResponse(xmlRoot): print(type(xmlRoot)) @@ -108,45 +124,42 @@ def getUserIdFromResponse(xmlRoot): """ Retrieves the ID from the expected XML response of the GetUserRequest """ - xpath = './soap:Body/ms_customer:GetUserResponse/ms_customer:User/ent:Id' + xpath = "./soap:Body/ms_customer:GetUserResponse/ms_customer:User/ent:Id" id_element = xmlRoot.find(xpath, namespaces) - if id_element is not None: return id_element.text else: return None - # or raise an exception, depending on your error handling strategy + # or raise an exception, depending on your error handling strategy -def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_token: str, authentication_token: str): +def callGetUserRequestAndRetrieveUserId( + client: AuthenticatedClient, developer_token: str, authentication_token: str +): """ Calls the GetUserRequest SOAP endpoint and retrieves the User ID from the response """ - payload = "\n \n " + developer_token + "\n " + authentication_token + "\n \n \n \n \n \n \n" + payload = ( + '\n \n ' + + developer_token + + "\n " + + authentication_token + + '\n \n \n \n \n \n \n' + ) client.uri = customer_manager_service_url - headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'GetUser' - } + headers = {"Content-Type": "text/xml", "SOAPAction": "GetUser"} request_params = SaaSRequestParams( - method=HTTPMethod.POST, - path="", - headers=headers, - body=payload - ) - response = client.send( - request_params + method=HTTPMethod.POST, path="", headers=headers, body=payload ) + response = client.send(request_params) context_logger = logger.bind( - **request_details( - client.get_authenticated_request(request_params), response - ) + **request_details(client.get_authenticated_request(request_params), response) ) user_id = getUserIdFromResponse(ElementTree.fromstring(response.text)) @@ -158,24 +171,23 @@ def callGetUserRequestAndRetrieveUserId(client: AuthenticatedClient , developer_ raise RequestFailureResponseException(response=response) - return user_id -def getAccountsIdFromResponse(xmlRoot ): +def getAccountsIdFromResponse(xmlRoot): """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts """ # Use XPath to directly find the Id element accounts_id = [] - xpath = './soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts' + xpath = "./soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts" accounts_element = xmlRoot.find(xpath, namespaces) - if(accounts_element is None): + if accounts_element is None: return None - xmlSubpath = './ent:Id' + xmlSubpath = "./ent:Id" for account_element in accounts_element: account_id = account_element.find(xmlSubpath, namespaces) if account_id is not None: @@ -184,69 +196,71 @@ def getAccountsIdFromResponse(xmlRoot ): return accounts_id - -def callGetAccountRequestAndRetrieveAccountsId(client: AuthenticatedClient , developer_token: str, authentication_token: str, user_id: str): +def callGetAccountRequestAndRetrieveAccountsId( + client: AuthenticatedClient, + developer_token: str, + authentication_token: str, + user_id: str, +): """ Calls the SearchAccounts SOAP endpoint and retrieves the Account ID from the response """ - payload = "\n\n \n \n "+ authentication_token + "\n " + developer_token + "\n \n \n \n \n \n UserId\n Equals\n "+ user_id +"\n \n \n \n \n 0\n 10\n \n \n \n\n" + payload = ( + '\n\n \n \n ' + + authentication_token + + '\n ' + + developer_token + + '\n \n \n \n \n \n UserId\n Equals\n ' + + user_id + + '\n \n \n \n \n 0\n 10\n \n \n \n\n' + ) - headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'SearchAccounts' - } + headers = {"Content-Type": "text/xml", "SOAPAction": "SearchAccounts"} client.uri = customer_manager_service_url request_params = SaaSRequestParams( - method=HTTPMethod.POST, - path="", - headers=headers, - body=payload + method=HTTPMethod.POST, path="", headers=headers, body=payload ) - response = client.send( - request_params - ) + response = client.send(request_params) context_logger = logger.bind( - **request_details( - client.get_authenticated_request(request_params), response - ) + **request_details(client.get_authenticated_request(request_params), response) ) accountIds = getAccountsIdFromResponse(ElementTree.fromstring(response.text)) if accountIds is None: context_logger.error( - "SearchAccounts request failed with the following message {}.", response.text + "SearchAccounts request failed with the following message {}.", + response.text, ) raise RequestFailureResponseException(response=response) context_logger.info( - "SearchAccounts request was succesfull with the following ids {}.", accountIds - ) + "SearchAccounts request was succesfull with the following ids {}.", accountIds + ) return accountIds -def getAudiencesIDsfromResponse(xmlRoot ): - +def getAudiencesIDsfromResponse(xmlRoot): """ Gets the Audience _ids from the GetAudiencesByIdsResponse """ audience_ids = [] - xpath = './soap:Body/ms_campaign:GetAudiencesByIdsResponse/ms_campaign:Audiences' + xpath = "./soap:Body/ms_campaign:GetAudiencesByIdsResponse/ms_campaign:Audiences" audiences_element = xmlRoot.find(xpath, namespaces) - if(audiences_element is None): + if audiences_element is None: return None for audience_leaf in audiences_element: - xmlSubpath = './ms_campaign:Id' + xmlSubpath = "./ms_campaign:Id" audience_id = audience_leaf.find(xmlSubpath, namespaces) if audience_id is not None: audience_ids.append(audience_id.text) @@ -254,35 +268,42 @@ def getAudiencesIDsfromResponse(xmlRoot ): return audience_ids - -def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, developer_token:str, authentication_token:str , user_id:str, account_id: str): - payload = "\n \n GetAudiencesByIds\n " + authentication_token + "\n " + account_id + "\n " + user_id + "\n " + developer_token + "\n \n \n \n CustomerList\n \n \n\n" +def callGetCustomerListAudiencesByAccounts( + client: AuthenticatedClient, + developer_token: str, + authentication_token: str, + user_id: str, + account_id: str, +): + payload = ( + '\n \n GetAudiencesByIds\n ' + + authentication_token + + '\n ' + + account_id + + '\n ' + + user_id + + '\n ' + + developer_token + + '\n \n \n \n CustomerList\n \n \n\n' + ) headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'GetAudiencesByIds', + "Content-Type": "text/xml", + "SOAPAction": "GetAudiencesByIds", } client.uri = campaing_manager_service_url request_params = SaaSRequestParams( - method=HTTPMethod.POST, - path="", - headers=headers, - body=payload + method=HTTPMethod.POST, path="", headers=headers, body=payload ) - response = client.send( - request_params - ) + response = client.send(request_params) context_logger = logger.bind( - **request_details( - client.get_authenticated_request(request_params), response - ) + **request_details(client.get_authenticated_request(request_params), response) ) - audiences_list = getAudiencesIDsfromResponse(ElementTree.fromstring(response.text)) if not audiences_list: @@ -293,7 +314,6 @@ def callGetCustomerListAudiencesByAccounts(client: AuthenticatedClient, develope return None - return audiences_list @@ -304,56 +324,95 @@ def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_emai """ base_filepath = "src/fides/api/service/saas_request/upload_files_templates/CustomerListRemoval.csv" - destination = "CustomerListRemoval.csv" - csv_headers = ["Type","Status","Id","Parent Id","Client Id","Modified Time","Name","Description","Scope","Audience","Action Type","SubType","Text"] + destination = "CustomerListRemoval.csv" + csv_headers = [ + "Type", + "Status", + "Id", + "Parent Id", + "Client Id", + "Modified Time", + "Name", + "Description", + "Scope", + "Audience", + "Action Type", + "SubType", + "Text", + ] - hashedEmail=hashlib.sha256(target_email.encode()).hexdigest() + hashedEmail = hashlib.sha256(target_email.encode()).hexdigest() shutil.copyfile(base_filepath, destination) - with open(destination,'a') as csvfile: - writer = csv.DictWriter(csvfile,csv_headers) + with open(destination, "a") as csvfile: + writer = csv.DictWriter(csvfile, csv_headers) for audience_id in audiences_ids: - writer.writerow({"Type": "Customer List", "Id": audience_id, "Client Id": "fides_ethyca", "Action Type": "Update" }) - writer.writerow({"Type": "Customer List Item", "Parent Id": audience_id, "Action Type": "Delete", "SubType": "Email", "Text": hashedEmail}) + writer.writerow( + { + "Type": "Customer List", + "Id": audience_id, + "Client Id": "fides_ethyca", + "Action Type": "Update", + } + ) + writer.writerow( + { + "Type": "Customer List Item", + "Parent Id": audience_id, + "Action Type": "Delete", + "SubType": "Email", + "Text": hashedEmail, + } + ) return destination -def getUploadURLFromResponse(xmlRoot ): +def getUploadURLFromResponse(xmlRoot): - xpath = './soap:Body/ms_campaign:GetBulkUploadUrlResponse/ms_campaign:UploadUrl' + xpath = "./soap:Body/ms_campaign:GetBulkUploadUrlResponse/ms_campaign:UploadUrl" upload_url_element = xmlRoot.find(xpath, namespaces) return upload_url_element.text -def getBulkUploadURL(client: AuthenticatedClient, developer_token: str, authentication_token: str, user_id: str, account_id: str): - - payload = "\n \n GetBulkUploadUrl\n " + authentication_token + "\n " + account_id + "\n " + user_id +"\n "+ developer_token + "\n \n \n \n ErrorsAndResults\n " + account_id + "\n \n \n\n" +def getBulkUploadURL( + client: AuthenticatedClient, + developer_token: str, + authentication_token: str, + user_id: str, + account_id: str, +): + + payload = ( + '\n \n GetBulkUploadUrl\n ' + + authentication_token + + '\n ' + + account_id + + '\n ' + + user_id + + '\n ' + + developer_token + + '\n \n \n \n ErrorsAndResults\n ' + + account_id + + "\n \n \n\n" + ) headers = { - 'Content-Type': 'text/xml', - 'SOAPAction': 'GetBulkUploadUrl', + "Content-Type": "text/xml", + "SOAPAction": "GetBulkUploadUrl", } client.uri = bulk_api_url request_params = SaaSRequestParams( - method=HTTPMethod.POST, - path="", - headers=headers, - body=payload - ) - - response = client.send( - request_params + method=HTTPMethod.POST, path="", headers=headers, body=payload ) + response = client.send(request_params) context_logger = logger.bind( - **request_details( - client.get_authenticated_request(request_params), response - ) + **request_details(client.get_authenticated_request(request_params), response) ) upload_url = getUploadURLFromResponse(ElementTree.fromstring(response.text)) @@ -364,32 +423,40 @@ def getBulkUploadURL(client: AuthenticatedClient, developer_token: str, authenti ) raise RequestFailureResponseException(response=response) - return upload_url ### Step 6 : Upload to the API -def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str ,developer_token: str, authentication_token: str, user_id: str, account_id: str ): +def bulkUploadCustomerList( + client: AuthenticatedClient, + url: str, + filepath: str, + developer_token: str, + authentication_token: str, + user_id: str, + account_id: str, +): headers = { - 'AuthenticationToken': authentication_token, - 'DeveloperToken': developer_token, - 'CustomerId': user_id, - 'AccountId': account_id} + "AuthenticationToken": authentication_token, + "DeveloperToken": developer_token, + "CustomerId": user_id, + "AccountId": account_id, + } - files=[ - ('uploadFile',('customerlist.csv',open(filepath,'rb'),'application/octet-stream')) + files = [ + ( + "uploadFile", + ("customerlist.csv", open(filepath, "rb"), "application/octet-stream"), + ) ] client.uri = url request_params = SaaSRequestParams( - method=HTTPMethod.POST, - headers = headers, - path="", - files=files + method=HTTPMethod.POST, headers=headers, path="", files=files ) response = client.send( @@ -397,9 +464,7 @@ def bulkUploadCustomerList(client: AuthenticatedClient, url: str, filepath: str ) context_logger = logger.bind( - **request_details( - client.get_authenticated_request(request_params), response - ) + **request_details(client.get_authenticated_request(request_params), response) ) parsedResponse = json.loads(response.text) diff --git a/tests/fixtures/saas/microsoft_advertising_fixtures.py b/tests/fixtures/saas/microsoft_advertising_fixtures.py index 0c909984f2..ec9e83689d 100644 --- a/tests/fixtures/saas/microsoft_advertising_fixtures.py +++ b/tests/fixtures/saas/microsoft_advertising_fixtures.py @@ -15,12 +15,18 @@ @pytest.fixture(scope="session") def microsoft_advertising_secrets(saas_config) -> Dict[str, Any]: return { - "domain": pydash.get(saas_config, "microsoft_advertising.domain") or secrets["domain"] , - "client_id" :pydash.get(saas_config, "microsoft_advertising.client_id") or secrets["client_id"] , - "client_secret" :pydash.get(saas_config, "microsoft_advertising.client_secret") or secrets["client_secret"] , - "dev_token": pydash.get(saas_config, "microsoft_advertising.dev_token") or secrets["dev_token"] , - 'access_token': pydash.get(saas_config, "microsoft_advertising.access_token") or secrets["access_token"] , - 'redirect_uri': pydash.get(saas_config, "microsoft_advertising.redirect_uri") or secrets["redirect_uri"] + "domain": pydash.get(saas_config, "microsoft_advertising.domain") + or secrets["domain"], + "client_id": pydash.get(saas_config, "microsoft_advertising.client_id") + or secrets["client_id"], + "client_secret": pydash.get(saas_config, "microsoft_advertising.client_secret") + or secrets["client_secret"], + "dev_token": pydash.get(saas_config, "microsoft_advertising.dev_token") + or secrets["dev_token"], + "access_token": pydash.get(saas_config, "microsoft_advertising.access_token") + or secrets["access_token"], + "redirect_uri": pydash.get(saas_config, "microsoft_advertising.redirect_uri") + or secrets["redirect_uri"], # add the rest of your secrets here } diff --git a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py index 82b507c1c2..719956e6e9 100644 --- a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py +++ b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py @@ -11,7 +11,7 @@ def test_connection( microsoft_advertising_runner: ConnectorRunner, policy: Policy, erasure_policy_string_rewrite: Policy, - microsoft_advertising_erasure_identity_email: str + microsoft_advertising_erasure_identity_email: str, ): microsoft_advertising_runner.test_connection() From 4e5bc7a5d80f1ec7fc5518046f19f962c154cd74 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 16:09:01 -0400 Subject: [PATCH 27/39] base defusedxm and Element import and funciton mapping --- ...microsoft_advertising_request_overrides.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index fded2a344a..f4396915ab 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -2,9 +2,11 @@ import hashlib import json import shutil -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional +from xml.etree.ElementTree import Element from defusedxml import ElementTree + from loguru import logger from fides.api.models.policy import Policy @@ -117,9 +119,7 @@ def microsoft_advertising_user_delete( return rows_updated -def getUserIdFromResponse(xmlRoot): - - print(type(xmlRoot)) +def getUserIdFromResponse(xmlRoot: Element) -> Optional[str] : """ Retrieves the ID from the expected XML response of the GetUserRequest @@ -129,14 +129,13 @@ def getUserIdFromResponse(xmlRoot): if id_element is not None: return id_element.text - else: - return None - # or raise an exception, depending on your error handling strategy + + return None def callGetUserRequestAndRetrieveUserId( client: AuthenticatedClient, developer_token: str, authentication_token: str -): +) -> str : """ Calls the GetUserRequest SOAP endpoint and retrieves the User ID from the response """ @@ -174,7 +173,7 @@ def callGetUserRequestAndRetrieveUserId( return user_id -def getAccountsIdFromResponse(xmlRoot): +def getAccountsIdFromResponse(xmlRoot: Element) -> Optional[List[Optional[str]]] : """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts @@ -201,7 +200,7 @@ def callGetAccountRequestAndRetrieveAccountsId( developer_token: str, authentication_token: str, user_id: str, -): +) -> List[Optional[str]] : """ Calls the SearchAccounts SOAP endpoint and retrieves the Account ID from the response """ @@ -247,7 +246,7 @@ def callGetAccountRequestAndRetrieveAccountsId( return accountIds -def getAudiencesIDsfromResponse(xmlRoot): +def getAudiencesIDsfromResponse(xmlRoot: Element) -> Optional[List[int]] : """ Gets the Audience _ids from the GetAudiencesByIdsResponse """ @@ -274,7 +273,7 @@ def callGetCustomerListAudiencesByAccounts( authentication_token: str, user_id: str, account_id: str, -): +) -> Optional[list[int]] : payload = ( '\n \n GetAudiencesByIds\n ' + authentication_token @@ -317,7 +316,7 @@ def callGetCustomerListAudiencesByAccounts( return audiences_list -def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_email: str): +def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_email: str) -> str: """ Createsa CSV with the values to remove the Customer List Objects. Since we dont know on Which Audience the Customer List Object is, we will remove it from all the Audiences @@ -369,12 +368,14 @@ def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_emai return destination -def getUploadURLFromResponse(xmlRoot): +def getUploadURLFromResponse(xmlRoot: Element) -> Optional[str] : xpath = "./soap:Body/ms_campaign:GetBulkUploadUrlResponse/ms_campaign:UploadUrl" upload_url_element = xmlRoot.find(xpath, namespaces) - return upload_url_element.text + if upload_url_element is not None: + return upload_url_element.text + return None def getBulkUploadURL( @@ -383,7 +384,7 @@ def getBulkUploadURL( authentication_token: str, user_id: str, account_id: str, -): +)-> str : payload = ( '\n \n GetBulkUploadUrl\n ' @@ -437,7 +438,7 @@ def bulkUploadCustomerList( authentication_token: str, user_id: str, account_id: str, -): +) -> bool : headers = { "AuthenticationToken": authentication_token, From a467a61b8b0c99783133717b2496c3fc586736d6 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 17:09:21 -0400 Subject: [PATCH 28/39] Adding dev token to the missing config --- data/saas/config/microsoft_advertising_config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/saas/config/microsoft_advertising_config.yml b/data/saas/config/microsoft_advertising_config.yml index 2aa839640f..15d58590bc 100644 --- a/data/saas/config/microsoft_advertising_config.yml +++ b/data/saas/config/microsoft_advertising_config.yml @@ -17,6 +17,10 @@ saas_config: label: Client secret description: Your Microsoft Advertising application's client secret sensitive: True + - name: dev_token + label: Developer token + description: Your Microsoft Advertising application's developer token + sensitive: True - name: redirect_uri label: Redirect URL description: The Fides URL to which users will be redirected upon successful authentication From 5310cf2d59c6483b04e1a6456cd53e4267bd09f5 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 17:32:08 -0400 Subject: [PATCH 29/39] FInalizing update import on the request override --- ...microsoft_advertising_request_overrides.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index f4396915ab..aa72eb4b8b 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -173,13 +173,13 @@ def callGetUserRequestAndRetrieveUserId( return user_id -def getAccountsIdFromResponse(xmlRoot: Element) -> Optional[List[Optional[str]]] : +def getAccountsIdFromResponse(xmlRoot: Element) -> Optional[List[str]] : """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts """ # Use XPath to directly find the Id element - accounts_id = [] + accounts_id: List[str] = [] xpath = "./soap:Body/ms_customer:SearchAccountsResponse/ms_customer:Accounts" accounts_element = xmlRoot.find(xpath, namespaces) @@ -190,17 +190,18 @@ def getAccountsIdFromResponse(xmlRoot: Element) -> Optional[List[Optional[str]]] for account_element in accounts_element: account_id = account_element.find(xmlSubpath, namespaces) if account_id is not None: - accounts_id.append(account_id.text) + appendTextFromElementToList(account_id, accounts_id) return accounts_id + def callGetAccountRequestAndRetrieveAccountsId( client: AuthenticatedClient, developer_token: str, authentication_token: str, user_id: str, -) -> List[Optional[str]] : +) -> List[str] : """ Calls the SearchAccounts SOAP endpoint and retrieves the Account ID from the response """ @@ -246,12 +247,12 @@ def callGetAccountRequestAndRetrieveAccountsId( return accountIds -def getAudiencesIDsfromResponse(xmlRoot: Element) -> Optional[List[int]] : +def getAudiencesIDsfromResponse(xmlRoot: Element) -> Optional[List[str]] : """ Gets the Audience _ids from the GetAudiencesByIdsResponse """ - audience_ids = [] + audience_ids: List[str] = [] xpath = "./soap:Body/ms_campaign:GetAudiencesByIdsResponse/ms_campaign:Audiences" audiences_element = xmlRoot.find(xpath, namespaces) @@ -262,7 +263,7 @@ def getAudiencesIDsfromResponse(xmlRoot: Element) -> Optional[List[int]] : xmlSubpath = "./ms_campaign:Id" audience_id = audience_leaf.find(xmlSubpath, namespaces) if audience_id is not None: - audience_ids.append(audience_id.text) + appendTextFromElementToList(audience_id, audience_ids) return audience_ids @@ -273,7 +274,7 @@ def callGetCustomerListAudiencesByAccounts( authentication_token: str, user_id: str, account_id: str, -) -> Optional[list[int]] : +) -> Optional[list[str]] : payload = ( '\n \n GetAudiencesByIds\n ' + authentication_token @@ -316,7 +317,7 @@ def callGetCustomerListAudiencesByAccounts( return audiences_list -def createCSVForRemovingCustomerListObject(audiences_ids: List[int], target_email: str) -> str: +def createCSVForRemovingCustomerListObject(audiences_ids: List[str], target_email: str) -> str: """ Createsa CSV with the values to remove the Customer List Objects. Since we dont know on Which Audience the Customer List Object is, we will remove it from all the Audiences @@ -483,3 +484,14 @@ def bulkUploadCustomerList( ) return True + + +def appendTextFromElementToList(element: Element, list: list[str]) -> None : + """ + Safely retrieves the text from the Element and appends it to the list + """ + if element is not None: + if element.text is not None: + list.append(element.text) + + return None From 511696b8ca49daa7c5f94d463a7f2d848b774dba Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 17:35:44 -0400 Subject: [PATCH 30/39] Adding removal of file after being used --- .../microsoft_advertising_request_overrides.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index aa72eb4b8b..5d11f0e049 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -2,6 +2,7 @@ import hashlib import json import shutil +import os from typing import Any, Dict, List, Optional from xml.etree.ElementTree import Element @@ -114,6 +115,8 @@ def microsoft_advertising_user_delete( account_id, ) + os.remove(csv_file) + rows_updated += 1 return rows_updated From 059b09f687027303da4ae981a7ced3b1799494b2 Mon Sep 17 00:00:00 2001 From: bruno Date: Mon, 26 Aug 2024 17:53:12 -0400 Subject: [PATCH 31/39] Updating Shared Schemas to comply with Pydantic V2 Upgrade --- src/fides/api/schemas/saas/shared_schemas.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/fides/api/schemas/saas/shared_schemas.py b/src/fides/api/schemas/saas/shared_schemas.py index ad5f0f76d9..bb866e40a6 100644 --- a/src/fides/api/schemas/saas/shared_schemas.py +++ b/src/fides/api/schemas/saas/shared_schemas.py @@ -26,13 +26,9 @@ class SaaSRequestParams(BaseModel): headers: Dict[str, Any] = {} query_params: Dict[str, Any] = {} body: Optional[str] = None + files: Optional[list] = None model_config = ConfigDict(use_enum_values=True) - files: Optional[list] - class Config: - """Using enum values""" - - use_enum_values = True class ConnectorParamRef(BaseModel): From c095a9e3e24be0c8b5aadb982e3d0f08601692ba Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 27 Aug 2024 10:28:39 -0400 Subject: [PATCH 32/39] Running Linters and static tests --- src/fides/api/schemas/saas/shared_schemas.py | 1 - ...microsoft_advertising_request_overrides.py | 101 ++++++++++-------- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/src/fides/api/schemas/saas/shared_schemas.py b/src/fides/api/schemas/saas/shared_schemas.py index b2f7c7057a..104e995f7c 100644 --- a/src/fides/api/schemas/saas/shared_schemas.py +++ b/src/fides/api/schemas/saas/shared_schemas.py @@ -30,7 +30,6 @@ class SaaSRequestParams(BaseModel): model_config = ConfigDict(use_enum_values=True) - class ConnectorParamRef(BaseModel): """A reference to a value in the connector params (by name)""" diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 5d11f0e049..cf51e670a8 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -1,13 +1,13 @@ import csv import hashlib import json -import shutil import os +import shutil from typing import Any, Dict, List, Optional - from xml.etree.ElementTree import Element -from defusedxml import ElementTree +# note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +from defusedxml import ElementTree from loguru import logger from fides.api.models.policy import Policy @@ -122,8 +122,7 @@ def microsoft_advertising_user_delete( return rows_updated -def getUserIdFromResponse(xmlRoot: Element) -> Optional[str] : - +def getUserIdFromResponse(xmlRoot: Element) -> Optional[str]: """ Retrieves the ID from the expected XML response of the GetUserRequest """ @@ -138,7 +137,7 @@ def getUserIdFromResponse(xmlRoot: Element) -> Optional[str] : def callGetUserRequestAndRetrieveUserId( client: AuthenticatedClient, developer_token: str, authentication_token: str -) -> str : +) -> str: """ Calls the GetUserRequest SOAP endpoint and retrieves the User ID from the response """ @@ -176,7 +175,7 @@ def callGetUserRequestAndRetrieveUserId( return user_id -def getAccountsIdFromResponse(xmlRoot: Element) -> Optional[List[str]] : +def getAccountsIdFromResponse(xmlRoot: Element) -> Optional[List[str]]: """ Retrieves the ID from the expected XML response of the SearchAccountsRequest TODO: Expand for Multiple accounts @@ -198,13 +197,12 @@ def getAccountsIdFromResponse(xmlRoot: Element) -> Optional[List[str]] : return accounts_id - def callGetAccountRequestAndRetrieveAccountsId( client: AuthenticatedClient, developer_token: str, authentication_token: str, user_id: str, -) -> List[str] : +) -> List[str]: """ Calls the SearchAccounts SOAP endpoint and retrieves the Account ID from the response """ @@ -250,7 +248,7 @@ def callGetAccountRequestAndRetrieveAccountsId( return accountIds -def getAudiencesIDsfromResponse(xmlRoot: Element) -> Optional[List[str]] : +def getAudiencesIDsfromResponse(xmlRoot: Element) -> Optional[List[str]]: """ Gets the Audience _ids from the GetAudiencesByIdsResponse """ @@ -277,7 +275,7 @@ def callGetCustomerListAudiencesByAccounts( authentication_token: str, user_id: str, account_id: str, -) -> Optional[list[str]] : +) -> Optional[list[str]]: payload = ( '\n \n GetAudiencesByIds\n ' + authentication_token @@ -320,7 +318,9 @@ def callGetCustomerListAudiencesByAccounts( return audiences_list -def createCSVForRemovingCustomerListObject(audiences_ids: List[str], target_email: str) -> str: +def createCSVForRemovingCustomerListObject( + audiences_ids: List[str], target_email: str +) -> str: """ Createsa CSV with the values to remove the Customer List Objects. Since we dont know on Which Audience the Customer List Object is, we will remove it from all the Audiences @@ -348,7 +348,7 @@ def createCSVForRemovingCustomerListObject(audiences_ids: List[str], target_emai shutil.copyfile(base_filepath, destination) - with open(destination, "a") as csvfile: + with open(destination, "a", encoding="utf8") as csvfile: writer = csv.DictWriter(csvfile, csv_headers) for audience_id in audiences_ids: writer.writerow( @@ -372,7 +372,7 @@ def createCSVForRemovingCustomerListObject(audiences_ids: List[str], target_emai return destination -def getUploadURLFromResponse(xmlRoot: Element) -> Optional[str] : +def getUploadURLFromResponse(xmlRoot: Element) -> Optional[str]: xpath = "./soap:Body/ms_campaign:GetBulkUploadUrlResponse/ms_campaign:UploadUrl" upload_url_element = xmlRoot.find(xpath, namespaces) @@ -388,7 +388,7 @@ def getBulkUploadURL( authentication_token: str, user_id: str, account_id: str, -)-> str : +) -> str: payload = ( '\n \n GetBulkUploadUrl\n ' @@ -442,7 +442,7 @@ def bulkUploadCustomerList( authentication_token: str, user_id: str, account_id: str, -) -> bool : +) -> bool: headers = { "AuthenticationToken": authentication_token, @@ -451,50 +451,57 @@ def bulkUploadCustomerList( "AccountId": account_id, } - files = [ - ( - "uploadFile", - ("customerlist.csv", open(filepath, "rb"), "application/octet-stream"), - ) - ] + ## using with open for memory allocation. See https://pylint.pycqa.org/en/latest/user_guide/messages/refactor/consider-using-with.html + with open(filepath, "rb") as file: + + upload_files = [ + ( + "uploadFile", + ( + "customerlist.csv", + file, + "application/octet-stream", + ), + ) + ] - client.uri = url + client.uri = url - request_params = SaaSRequestParams( - method=HTTPMethod.POST, headers=headers, path="", files=files - ) + request_params = SaaSRequestParams( + method=HTTPMethod.POST, headers=headers, path="", files=upload_files + ) - response = client.send( - request_params, - ) + response = client.send( + request_params, + ) - context_logger = logger.bind( - **request_details(client.get_authenticated_request(request_params), response) - ) + context_logger = logger.bind( + **request_details( + client.get_authenticated_request(request_params), response + ) + ) - parsedResponse = json.loads(response.text) + parsedResponse = json.loads(response.text) - if not parsedResponse["TrackingId"]: - context_logger.error( - "GetBulkUploadUrl collected No upload URL {}.", response.text - ) - raise RequestFailureResponseException(response=response) + if not parsedResponse["TrackingId"]: + context_logger.error( + "GetBulkUploadUrl collected No upload URL {}.", response.text + ) + raise RequestFailureResponseException(response=response) - ## Do we need a process to check the status of the Upload? - ## How are we persisting data? - context_logger.info( - "Tracking ID of the recent upload: {}.", parsedResponse["TrackingId"] - ) + ## Do we need a process to check the status of the Upload? + ## How are we persisting data? + context_logger.info( + "Tracking ID of the recent upload: {}.", parsedResponse["TrackingId"] + ) return True -def appendTextFromElementToList(element: Element, list: list[str]) -> None : +def appendTextFromElementToList(element: Element, list_of_elements: List[str]) -> None: """ Safely retrieves the text from the Element and appends it to the list """ if element is not None: if element.text is not None: - list.append(element.text) - - return None + list_of_elements.append(element.text) From c1bef90ceae263e72ec28c91dd80e3cffd2a2d9a Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 27 Aug 2024 10:44:33 -0400 Subject: [PATCH 33/39] Adding Types stub so mypy can recognize the defusedXML library --- requirements.txt | 1 + .../microsoft_advertising_request_overrides.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0c9628f954..a2a6d0d591 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ cryptography==42.0.0 dask==2022.9.2 deepdiff==6.3.0 defusedxml==0.7.1 +types-defusedxml==0.7.0.20240218 expandvars==0.9.0 fastapi[all]==0.111.0 fastapi-pagination[sqlalchemy]==0.12.25 diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index cf51e670a8..fcb80010d7 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -6,7 +6,6 @@ from typing import Any, Dict, List, Optional from xml.etree.ElementTree import Element -# note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports from defusedxml import ElementTree from loguru import logger From cd6e05745b330792e90cbd380365491c81782d53 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 28 Aug 2024 11:36:35 -0400 Subject: [PATCH 34/39] Remove Sandbox References --- data/saas/config/microsoft_advertising_config.yml | 13 ++----------- .../microsoft_advertising_request_overrides.py | 4 ---- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/data/saas/config/microsoft_advertising_config.yml b/data/saas/config/microsoft_advertising_config.yml index 15d58590bc..0a70d76cbc 100644 --- a/data/saas/config/microsoft_advertising_config.yml +++ b/data/saas/config/microsoft_advertising_config.yml @@ -8,8 +8,8 @@ saas_config: connector_params: - name: domain - default_value: clientcenter.api.sandbox.bingads.microsoft.com # change to the prod URL for the default - description: The base URL for your Microsoft Advertising + default_value: login.microsoftonline.com + description: The base URL for Microsoft Advertising Login Authentication - name: client_id label: Client ID description: Your Microsoft Advertising application's client ID @@ -34,9 +34,6 @@ saas_config: expires_in: 3600 authorization_request: method: GET - client_config: - protocol: https - host: login.microsoftonline.com path: /consumers/oauth2/v2.0/authorize query_params: - name: client_id @@ -51,9 +48,6 @@ saas_config: value: token_request: method: POST - client_config: - protocol: https - host: login.microsoftonline.com path: /consumers/oauth2/v2.0/token headers: - name: Content-Type @@ -68,9 +62,6 @@ saas_config: } refresh_request: method: POST - client_config: - protocol: https - host: login.microsoftonline.com path: /consumers/oauth2/v2.0/token headers: - name: Content-Type diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index fcb80010d7..758ad1c52f 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -22,10 +22,6 @@ ) from fides.api.util.logger_context_utils import request_details -sandbox_customer_manager_service_url = "https://clientcenter.api.sandbox.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" -sandbox_campaing_manager_service_url = "https://campaign.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" -sandbox_bulk_api_url = "https://bulk.api.sandbox.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc?wsdl" - customer_manager_service_url = "https://clientcenter.api.bingads.microsoft.com/Api/CustomerManagement/v13/CustomerManagementService.svc" campaing_manager_service_url = "https://campaign.api.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/CampaignManagementService.svc" bulk_api_url = "https://bulk.api.bingads.microsoft.com/Api/Advertiser/CampaignManagement/v13/BulkService.svc" From 5dc3ace41e2954ddd838a1859fe355133590f943 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 28 Aug 2024 11:40:24 -0400 Subject: [PATCH 35/39] Removing Sandbox Input on the base user delete --- .../microsoft_advertising_request_overrides.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py index 758ad1c52f..29bdd1c619 100644 --- a/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/microsoft_advertising_request_overrides.py @@ -62,7 +62,6 @@ def microsoft_advertising_user_delete( policy: Policy, privacy_request: PrivacyRequest, secrets: Dict[str, Any], - is_sandbox: bool = False, ) -> int: """ Process of removing an User email from the Microsoft Advertising Platform From 57b274dd44e09ef6febfc2001490ce8305ab978b Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 28 Aug 2024 11:53:07 -0400 Subject: [PATCH 36/39] Setting up a proper Type for files. Adding Arbitrary types --- src/fides/api/schemas/saas/shared_schemas.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fides/api/schemas/saas/shared_schemas.py b/src/fides/api/schemas/saas/shared_schemas.py index 104e995f7c..fea74a13d8 100644 --- a/src/fides/api/schemas/saas/shared_schemas.py +++ b/src/fides/api/schemas/saas/shared_schemas.py @@ -1,8 +1,11 @@ from enum import Enum -from typing import Any, Dict, Optional +from io import BufferedReader +from typing import Any, Dict, Optional, Tuple, List from pydantic import BaseModel, ConfigDict +RequestFile = Tuple[str, Tuple[str, BufferedReader, str]] + class HTTPMethod(Enum): """Enum to represent HTTP Methods""" @@ -26,8 +29,8 @@ class SaaSRequestParams(BaseModel): headers: Dict[str, Any] = {} query_params: Dict[str, Any] = {} body: Optional[str] = None - files: Optional[list] = None - model_config = ConfigDict(use_enum_values=True) + files: Optional[List[RequestFile]] = None + model_config = ConfigDict(use_enum_values=True, arbitrary_types_allowed=True) class ConnectorParamRef(BaseModel): From b63e0257b2723d5d8467e752180a1a803688e857 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 28 Aug 2024 12:15:09 -0400 Subject: [PATCH 37/39] Removing unnecesary parameters --- .../integration_tests/saas/test_microsoft_advertising_task.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py index 719956e6e9..a164d10db4 100644 --- a/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py +++ b/tests/ops/integration_tests/saas/test_microsoft_advertising_task.py @@ -4,14 +4,12 @@ from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") @pytest.mark.integration_saas class TestMicrosoftAdvertisingConnector: def test_connection( self, microsoft_advertising_runner: ConnectorRunner, - policy: Policy, - erasure_policy_string_rewrite: Policy, - microsoft_advertising_erasure_identity_email: str, ): microsoft_advertising_runner.test_connection() From 488c78cf7905b56c7f7c7076d7f86b3d033f89d4 Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 28 Aug 2024 12:16:41 -0400 Subject: [PATCH 38/39] Updating Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1945ae355d..8c644fdbfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The types of changes are: ### Added - Added Gzip Middleware for responses [#5225](https://github.com/ethyca/fides/pull/5225) - Adding source and submitted_by fields to privacy requests (Fidesplus) [#5206](https://github.com/ethyca/fides/pull/5206) +- Adding erasure support for Microsoft Advertising [#5197](https://github.com/ethyca/fides/pull/5197) ### Changed - Removed unused `username` parameter from the Delighted integration configuration [#5220](https://github.com/ethyca/fides/pull/5220) From 203e887bc900bf06234c28e4fdc3bc6a6a061d2f Mon Sep 17 00:00:00 2001 From: bruno Date: Wed, 28 Aug 2024 12:21:47 -0400 Subject: [PATCH 39/39] Uploading Import order on shared schema --- src/fides/api/schemas/saas/shared_schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/schemas/saas/shared_schemas.py b/src/fides/api/schemas/saas/shared_schemas.py index fea74a13d8..98f0ae3e36 100644 --- a/src/fides/api/schemas/saas/shared_schemas.py +++ b/src/fides/api/schemas/saas/shared_schemas.py @@ -1,6 +1,6 @@ from enum import Enum from io import BufferedReader -from typing import Any, Dict, Optional, Tuple, List +from typing import Any, Dict, List, Optional, Tuple from pydantic import BaseModel, ConfigDict