Merge pull request #19 from dynatrace-oss/APM-302265-Update-mapping-f…

…or-azure-activity-log-with-new-attributes APM-302265: Update mapping for Azure Activity Logs
dynatrace-oss · Jun 9, 2021 · 411ea4e · 411ea4e
2 parents bc190d3 + 6250e31
commit 411ea4e
Show file tree

Hide file tree

Showing 4 changed files with 146 additions and 18 deletions.
diff --git a/logs_ingest/config/activity_logs.json b/logs_ingest/config/activity_logs.json
@@ -22,6 +22,14 @@
         {
           "key": "content",
           "pattern": "resultDescription"
+        },
+        {
+          "key": "audit.action",
+          "pattern": "operationName"
+        },
+        {
+          "key": "audit.result",
+          "pattern": "replace_regex(resultType,'(.*)\\.$','$1') || resultType"
         }
       ]
     },
@@ -45,6 +53,18 @@
         {
           "key": "content",
           "pattern": "if(properties.statusMessage == null, &properties.message, &properties.statusMessage, @)"
+        },
+        {
+          "key": "audit.identity",
+          "pattern": "identity.claims.\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name\""
+        },
+        {
+          "key": "audit.action",
+          "pattern": "operationName"
+        },
+        {
+          "key": "audit.result",
+          "pattern": "replace_regex(resultSignature,'(.*)\\.$','$1') || resultSignature"
         }
       ]
     },
@@ -68,6 +88,49 @@
         {
           "key": "content",
           "pattern": "properties.eventProperties"
+        },
+        {
+          "key": "audit.action",
+          "pattern": "operationName"
+        },
+        {
+          "key": "audit.result",
+          "pattern": "replace_regex(resultType,'(.*)\\.$','$1') || resultType"
+        }
+      ]
+    },
+    {
+      "sources": [
+        {
+          "sourceType": "logs",
+          "source": "category",
+          "condition": "$eq('Policy')"
+        }
+      ],
+      "attributes": [
+        {
+          "key": "timestamp",
+          "pattern": "time"
+        },
+        {
+          "key": "log.source",
+          "pattern": "join('', ['Activity Log - ', category])"
+        },
+        {
+          "key": "content",
+          "pattern": "properties.message"
+        },
+        {
+          "key": "audit.identity",
+          "pattern": "\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name\""
+        },
+        {
+          "key": "audit.action",
+          "pattern": "operationName"
+        },
+        {
+          "key": "audit.result",
+          "pattern": "replace_regex(resultSignature,'(.*)\\.$','$1') || resultSignature"
         }
       ]
     }

diff --git a/logs_ingest/config/default.json b/logs_ingest/config/default.json
@@ -20,6 +20,14 @@
         {
           "key": "content",
           "pattern": "properties"
+        },
+        {
+          "key": "audit.action",
+          "pattern": "if(properties.eventCategory, &operationName, &null, @)"
+        },
+        {
+          "key": "audit.result",
+          "pattern": "if(properties.eventCategory, &(replace_regex(resultType,'(.*)\\.$','$1') || resultType), &null, @)"
         }
       ]
     }

diff --git a/tests/unit/extraction_rules_test/test_activity_logs.py b/tests/unit/extraction_rules_test/test_activity_logs.py
@@ -22,13 +22,13 @@
     SUBSCRIPTION_ATTRIBUTE, RESOURCE_ID_ATTRIBUTE
 from logs_ingest.self_monitoring import SelfMonitoring
 
-activity_record = {
+alert_record = {
     "time": "2021-02-09T11:15:57.0501894Z",
     "resourceId": "/SUBSCRIPTIONS/69B51384-146C-4685-9DAB-5AE01877D7B8/RESOURCEGROUPS/DTMAWO/PROVIDERS/MICROSOFT.INSIGHTS/ACTIVITYLOGALERTS/AA - ALERT ADMINISTRACYJNY",
     "correlationId": "d9381714-0c92-49e4-b471-d67199f857c0",
     "operationName": "Microsoft.Insights/ActivityLogAlerts/Activated/action",
     "level": "Information",
-    "resultType": "Succeeded",
+    "resultType": "Succeeded.",
     "resultDescription": "Alert: AA - Alert Administracyjny called on action groups : sendwebhook",
     "category": "Alert",
     "properties": {
@@ -45,7 +45,7 @@
     }
 }
 
-activity_expected_output = {
+alert_expected_output = {
     "cloud.provider": "Azure",
     "timestamp": "2021-02-09T11:15:57.0501894Z",
     "log.source": "Activity Log - Alert",
@@ -55,7 +55,9 @@
     SUBSCRIPTION_ATTRIBUTE: "69B51384-146C-4685-9DAB-5AE01877D7B8",
     RESOURCE_GROUP_ATTRIBUTE: "DTMAWO",
     RESOURCE_TYPE_ATTRIBUTE: "MICROSOFT.INSIGHTS/ACTIVITYLOGALERTS",
-    RESOURCE_NAME_ATTRIBUTE: "AA - ALERT ADMINISTRACYJNY"
+    RESOURCE_NAME_ATTRIBUTE: "AA - ALERT ADMINISTRACYJNY",
+    "audit.action": "Microsoft.Insights/ActivityLogAlerts/Activated/action",
+    "audit.result": "Succeeded"
 }
 
 administrative_record = {
@@ -87,17 +89,19 @@
             "iat": "1612785544",
             "nbf": "1612785544",
             "exp": "1612872244",
-            "aio": "E2ZgYDAQS/prW6b0Zsah6KMXtnTEAQA=",
-            "appid": "80369ed6-5f11-4dd9-bef3-692475845e77",
-            "appidacr": "2",
-            "http://schemas.microsoft.com/identity/claims/identityprovider": "https://sts.windows.net/70ebe3a3-5b30-435d-9d67-7716d74ca190/",
-            "http://schemas.microsoft.com/identity/claims/objectidentifier": "e7018f64-88e2-46af-a197-7b9084d8346a",
-            "rh": "0.AAAAo-PrcDBbXUOdZ3cW10yhkNaeNoARX9lNvvNpJHWEXndFAAA.",
-            "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": "e7018f64-88e2-46af-a197-7b9084d8346a",
-            "http://schemas.microsoft.com/identity/claims/tenantid": "70ebe3a3-5b30-435d-9d67-7716d74ca190",
+            "aio": "AUQAu/8TAAAAlSdddOpuxGAphkybH3N4EdIz5xuTrAwxum1uL6e+FO03x2G20rOQD3KvxRiAhzEAPcXk61pJ4Tsv6IzQ9phcsA==",
+            "appid": "04b07795-8ddb-461a-bbee-02f9e1bf7b46",
+            "appidacr": "0",
+            "http://schemas.microsoft.com/2012/01/devicecontext/claims/identifier": "7ey32c5a-9347-4f4c-8518-178b7cc0c1b7",
+            "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname": "Kowalski",
+            "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname": "Jan",
             "uti": "gVQtWBk9fkuRyF80Tn1XAA",
             "ver": "1.0",
-            "xms_tcdt": "1415644249"
+            "xms_tcdt": "1415644249",
+            "http://schemas.microsoft.com/identity/claims/scope": "user_impersonation",
+            "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": "SQDfplmjvs2zlBolO2iAHUNgznOepOpteDiCAs0",
+            "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name": "[email protected]",
+            "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn": "[email protected]",
         }
     },
     "level": "Error",
@@ -107,7 +111,7 @@
         "statusMessage": "{\"error\":{\"code\":\"ResourceGroupNotFound\",\"message\":\"Resource group 'mw-gr1' could not be found.\"}}",
         "eventCategory": "Administrative",
         "entity": "/SUBSCRIPTIONS/69B51384-146C-4685-9DAB-5AE01877D7B8/RESOURCEGROUPS/MW-GR1/PROVIDERS/MICROSOFT.STORAGE/STORAGEACCOUNTS/DTMWSTORAGE1",
-        "message": "Microsoft.Storage/storageAccounts/listAccountSas/action",
+        "message": "MICROSOFT.STORAGE/STORAGEACCOUNTS/LISTACCOUNTSAS/ACTION",
         "hierarchy": "70ebe3a3-5b30-435d-9d67-7716d74ca190/GroupLevel0/GroupALevel1/GroupAALevel2/69b51384-146c-4685-9dab-5ae01877d7b8"
     }
 }
@@ -124,6 +128,9 @@
     RESOURCE_TYPE_ATTRIBUTE: "MICROSOFT.STORAGE/STORAGEACCOUNTS",
     RESOURCE_NAME_ATTRIBUTE: "DTMWSTORAGE1",
     "dt.source_entity": "AZURE_STORAGE_ACCOUNT-A5A9F4A68D9B0D44",
+    "audit.identity": "[email protected]",
+    "audit.action": "MICROSOFT.STORAGE/STORAGEACCOUNTS/LISTACCOUNTSAS/ACTION",
+    "audit.result": "Failed.NotFound"
 }
 
 policy_record = {
@@ -186,13 +193,15 @@
     "timestamp": "2021-02-09T08:45:27.3186996Z",
     "log.source": "Activity Log - Policy",
     "severity": "Warning",
-    "content": json.dumps(policy_record["properties"]),
+    "content": "Microsoft.Authorization/policies/audit/action",
     RESOURCE_ID_ATTRIBUTE: "/SUBSCRIPTIONS/97E9B03F-04D6-4B69-B307-35F483F7ED81/RESOURCEGROUPS/AZUREBATCH-CB989D3E-4BCC-4ABC-BC32-D1B6DFA2E6B0-C/PROVIDERS/MICROSOFT.COMPUTE/VIRTUALMACHINESCALESETS/CB989D3E-4BCC-4ABC-BC32-D1B6DFA2E6B0-AZUREBATCH-VMSS-D",
     SUBSCRIPTION_ATTRIBUTE: "97E9B03F-04D6-4B69-B307-35F483F7ED81",
     RESOURCE_GROUP_ATTRIBUTE: "AZUREBATCH-CB989D3E-4BCC-4ABC-BC32-D1B6DFA2E6B0-C",
     RESOURCE_TYPE_ATTRIBUTE: "MICROSOFT.COMPUTE/VIRTUALMACHINESCALESETS",
     RESOURCE_NAME_ATTRIBUTE: "CB989D3E-4BCC-4ABC-BC32-D1B6DFA2E6B0-AZUREBATCH-VMSS-D",
     "dt.source_entity": "AZURE_VM_SCALE_SET-E65F5FB7B1076140",
+    "audit.action": "MICROSOFT.AUTHORIZATION/POLICIES/AUDIT/ACTION",
+    "audit.result": "Succeeded"
 }
 
 resource_health_record = {
@@ -228,6 +237,8 @@
     RESOURCE_TYPE_ATTRIBUTE: "MICROSOFT.NETWORK/LOADBALANCERS",
     RESOURCE_NAME_ATTRIBUTE: "KUBERNETES",
     "dt.source_entity": "AZURE_LOAD_BALANCER-0C3A32CD8FA39936",
+    "audit.action": "Microsoft.Resourcehealth/healthevent/Updated/action",
+    "audit.result": "Updated"
 }
 
 
@@ -236,9 +247,9 @@ def self_monitoring():
     return SelfMonitoring(execution_time=datetime.utcnow())
 
 
-def test_activity_log(self_monitoring):
-    actual_output = parse_record(activity_record, self_monitoring)
-    assert actual_output == activity_expected_output
+def test_alert_log(self_monitoring):
+    actual_output = parse_record(alert_record, self_monitoring)
+    assert actual_output == alert_expected_output
 
 
 def test_administrative_log(self_monitoring):

diff --git a/tests/unit/extraction_rules_test/test_default.py b/tests/unit/extraction_rules_test/test_default.py
@@ -82,12 +82,58 @@
     RESOURCE_NAME_ATTRIBUTE: "WEAT"
 }
 
+default_activity_log = {
+    "time": "2021-02-09T09:57:26.498Z",
+    "correlationId": "55dc161c-c72e-4cb7-b41a-4e7e6835c076",
+    "operationName": "Microsoft.ServiceHealth/actionrequired/action",
+    "level": "Information",
+    "resultType": "Activated",
+    "category": "ServiceHealth",
+    "properties": {
+        "eventCategory": "ServiceHealth",
+        "eventProperties": {
+            "title": "Action Required: Security Advisory on Linux Kernel TCP vulnerabilities",
+            "service": "Virtual Machines",
+            "region": "Australia Central",
+            "communication": "<p>Microsoft Azure is aware of the disclosure of three severe Linux kernel TCP networking vulnerabilities, known as TCP SACK Panic:</p><ul><li><a href=\"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11477\">CVE-2019-11477</a></li><li><a href=\"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11478\">CVE-2019-11478</a></li><li><a href=\"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11479\">CVE-2019-11479</a></li></ul><p>Virtual Machines running any Linux distribution should be addressed.&nbsp; For guidance on these vulnerabilities,&nbsp;please refer to the Linux support channels for your distribution or go to our <a href=\"https://portal.msrc.microsoft.com/en-US/security-guidance/advisory/ADV190020\">Microsoft Security Advisory 190020</a>.</p><p>Note: Virtual Machines running Windows are not affected by this vulnerability.</p><p><strong>Recommended Actions:</strong></p><ul><li>If you are running a Linux kernel in your Azure environment, you should contact the provider of that Linux kernel to understand their recommendation for protecting your installation. </li><li>Refer to the <a href=\"https://portal.msrc.microsoft.com/en-US/security-guidance/advisory/ADV190020\">Microsoft Security Advisory 190020</a> for\n     guidance around specific Azure services (Azure Sphere, Azure Kubernetes\n     Service, and Azure HDInsight)\n\n\n</li></ul><p></p>",
+            "incidentType": "ActionRequired",
+            "trackingId": "GTK4-188",
+            "impactStartTime": "2019-06-28T00:00:00Z",
+            "impactedServices": "[{\"ImpactedRegions\":[{\"RegionName\":\"Australia Central\"},{\"RegionName\":\"Australia Central 2\"},{\"RegionName\":\"Australia East\"},{\"RegionName\":\"Australia Southeast\"},{\"RegionName\":\"Brazil South\"},{\"RegionName\":\"Canada Central\"},{\"RegionName\":\"Canada East\"},{\"RegionName\":\"Central India\"},{\"RegionName\":\"Central US\"},{\"RegionName\":\"Central US EUAP\"},{\"RegionName\":\"East Asia\"},{\"RegionName\":\"East US\"},{\"RegionName\":\"East US 2\"},{\"RegionName\":\"East US 2 EUAP\"},{\"RegionName\":\"France Central\"},{\"RegionName\":\"France South\"},{\"RegionName\":\"Japan East\"},{\"RegionName\":\"Japan West\"},{\"RegionName\":\"Korea Central\"},{\"RegionName\":\"Korea South\"},{\"RegionName\":\"North Central US\"},{\"RegionName\":\"North Europe\"},{\"RegionName\":\"South Africa North\"},{\"RegionName\":\"South Africa West\"},{\"RegionName\":\"South Central US\"},{\"RegionName\":\"Southeast Asia\"},{\"RegionName\":\"South India\"},{\"RegionName\":\"UAE Central\"},{\"RegionName\":\"UAE North\"},{\"RegionName\":\"UK South\"},{\"RegionName\":\"UK West\"},{\"RegionName\":\"West Central US\"},{\"RegionName\":\"West Europe\"},{\"RegionName\":\"West India\"},{\"RegionName\":\"West US\"},{\"RegionName\":\"West US 2\"}],\"ServiceName\":\"Virtual Machines\"},{\"ImpactedRegions\":[{\"RegionName\":\"Australia Central\"},{\"RegionName\":\"Australia Central 2\"},{\"RegionName\":\"Australia East\"},{\"RegionName\":\"Australia Southeast\"},{\"RegionName\":\"Brazil South\"},{\"RegionName\":\"Canada Central\"},{\"RegionName\":\"Canada East\"},{\"RegionName\":\"Central India\"},{\"RegionName\":\"Central US\"},{\"RegionName\":\"Central US EUAP\"},{\"RegionName\":\"East Asia\"},{\"RegionName\":\"East US\"},{\"RegionName\":\"East US 2\"},{\"RegionName\":\"East US 2 EUAP\"},{\"RegionName\":\"France Central\"},{\"RegionName\":\"France South\"},{\"RegionName\":\"Japan East\"},{\"RegionName\":\"Japan West\"},{\"RegionName\":\"Korea Central\"},{\"RegionName\":\"Korea South\"},{\"RegionName\":\"North Central US\"},{\"RegionName\":\"North Europe\"},{\"RegionName\":\"South Africa North\"},{\"RegionName\":\"South Africa West\"},{\"RegionName\":\"South Central US\"},{\"RegionName\":\"Southeast Asia\"},{\"RegionName\":\"South India\"},{\"RegionName\":\"UAE Central\"},{\"RegionName\":\"UAE North\"},{\"RegionName\":\"UK South\"},{\"RegionName\":\"UK West\"},{\"RegionName\":\"West Central US\"},{\"RegionName\":\"West Europe\"},{\"RegionName\":\"West India\"},{\"RegionName\":\"West US\"},{\"RegionName\":\"West US 2\"}],\"ServiceName\":\"Virtual Machine Scale Sets\"}]",
+            "impactedServicesTableRows": "<tr>\r\n<td align='center' style='padding: 5px 10px; border-right:1px solid black; border-bottom:1px solid black'>Virtual Machines</td>\r\n<td align='center' style='padding: 5px 10px; border-bottom:1px solid black'>Australia Central<br>Australia Central 2<br>Australia East<br>Australia Southeast<br>Brazil South<br>Canada Central<br>Canada East<br>Central India<br>Central US<br>Central US EUAP<br>East Asia<br>East US<br>East US 2<br>East US 2 EUAP<br>France Central<br>France South<br>Japan East<br>Japan West<br>Korea Central<br>Korea South<br>North Central US<br>North Europe<br>South Africa North<br>South Africa West<br>South Central US<br>Southeast Asia<br>South India<br>UAE Central<br>UAE North<br>UK South<br>UK West<br>West Central US<br>West Europe<br>West India<br>West US<br>West US 2<br></td>\r\n</tr>\r\n<tr>\r\n<td align='center' style='padding: 5px 10px; border-right:1px solid black; border-bottom:1px solid black'>Virtual Machine Scale Sets</td>\r\n<td align='center' style='padding: 5px 10px; border-bottom:1px solid black'>Australia Central<br>Australia Central 2<br>Australia East<br>Australia Southeast<br>Brazil South<br>Canada Central<br>Canada East<br>Central India<br>Central US<br>Central US EUAP<br>East Asia<br>East US<br>East US 2<br>East US 2 EUAP<br>France Central<br>France South<br>Japan East<br>Japan West<br>Korea Central<br>Korea South<br>North Central US<br>North Europe<br>South Africa North<br>South Africa West<br>South Central US<br>Southeast Asia<br>South India<br>UAE Central<br>UAE North<br>UK South<br>UK West<br>West Central US<br>West Europe<br>West India<br>West US<br>West US 2<br></td>\r\n</tr>\r\n",
+            "defaultLanguageTitle": "Action Required: Security Advisory on Linux Kernel TCP vulnerabilities",
+            "defaultLanguageContent": "<p>Microsoft Azure is aware of the disclosure of three severe Linux kernel TCP networking vulnerabilities, known as TCP SACK Panic:</p><ul><li><a href=\"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11477\">CVE-2019-11477</a></li><li><a href=\"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11478\">CVE-2019-11478</a></li><li><a href=\"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11479\">CVE-2019-11479</a></li></ul><p>Virtual Machines running any Linux distribution should be addressed.&nbsp; For guidance on these vulnerabilities,&nbsp;please refer to the Linux support channels for your distribution or go to our <a href=\"https://portal.msrc.microsoft.com/en-US/security-guidance/advisory/ADV190020\">Microsoft Security Advisory 190020</a>.</p><p>Note: Virtual Machines running Windows are not affected by this vulnerability.</p><p><strong>Recommended Actions:</strong></p><ul><li>If you are running a Linux kernel in your Azure environment, you should contact the provider of that Linux kernel to understand their recommendation for protecting your installation. </li><li>Refer to the <a href=\"https://portal.msrc.microsoft.com/en-US/security-guidance/advisory/ADV190020\">Microsoft Security Advisory 190020</a> for\n     guidance around specific Azure services (Azure Sphere, Azure Kubernetes\n     Service, and Azure HDInsight)\n\n\n</li></ul><p></p>",
+            "stage": "Active",
+            "communicationId": "11000026609725",
+            "endTime": "2019-07-04T00:00:00Z",
+            "isHIR": "false",
+            "version": "0.1.1"
+        }
+    }
+}
+
+expected_output_activity_log = {
+    "cloud.provider": "Azure",
+    "timestamp": "2021-02-09T09:57:26.498Z",
+    "log.source": "Activity Log - ServiceHealth",
+    "severity": "Information",
+    "content": json.dumps(default_activity_log["properties"]),
+    "audit.action": "Microsoft.ServiceHealth/actionrequired/action",
+    "audit.result": "Activated"
+}
+
 
 def test_default():
     actual_output = main.parse_record(record, SelfMonitoring(execution_time=datetime.utcnow()))
     assert actual_output == expected_output
 
+
 def test_trimming_attribute_values(monkeypatch: MonkeyPatchFixture):
     monkeypatch.setattr(main, 'attribute_value_length_limit', 4)
     actual_output = main.parse_record(record, SelfMonitoring(execution_time=datetime.utcnow()))
     assert actual_output == expected_output_attribute_values_trimmed
+
+
+def test_default_activity_log():
+    actual_output = main.parse_record(default_activity_log, SelfMonitoring(execution_time=datetime.utcnow()))
+    assert actual_output == expected_output_activity_log