From 732619532456210df2bfa0860490d6a0515c3d5a Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Wed, 7 Jul 2021 15:30:39 -0500 Subject: [PATCH] Add network watcher enablement when safe (#22) * Add network watcher enablement when safe Signed-off-by: Chad Kittel * Fix typo in param description. Signed-off-by: Chad Kittel --- docs/additional-considerations.md | 2 + docs/deploy/04-subscription.md | 19 ++++- docs/deploy/14-cleanup.md | 4 +- subscription.json | 116 +++++++++++++++++++++++++++++- 4 files changed, 136 insertions(+), 5 deletions(-) diff --git a/docs/additional-considerations.md b/docs/additional-considerations.md index 3d4ca8f7..a1d127f1 100644 --- a/docs/additional-considerations.md +++ b/docs/additional-considerations.md @@ -38,6 +38,8 @@ In addition to Network Watcher aiding in compliance considerations, it's also a If you do not have Network Watchers and NSG Flow Logs enabled on your subscription, consider doing so via Azure Policy at the Subscription or Management Group level to provide consistent naming and region selection. See the [Deploy network watcher when virtual networks are created](https://portal.azure.com/#blade/Microsoft_Azure_Policy/PolicyDetailBlade/definitionId/%2Fproviders%2FMicrosoft.Authorization%2FpolicyDefinitions%2Fa9b99dd8-06c5-4317-8629-9d86a3c6e7d9) policy combined with the [Flow logs should be enabled for every network security group](https://portal.azure.com/#blade/Microsoft_Azure_Policy/PolicyDetailBlade/definitionId/%2Fproviders%2FMicrosoft.Authorization%2FpolicyDefinitions%2F27960feb-a23c-4577-8d36-ef8b5f35e0be) policy. +The [subscription.json](../subscription.json) file does include related Network Watcher policies, and will attempt to deploy Network Watcher if you do not already have evidence of them in your subscription. Reference those policy implementations if you would like to evaluate your current Network Watcher deployment strategy. + ### More strict Network Security Groups (NSGs) The NSGs that exist around the cluster node pool subnets specifically block any SSH access attempts only allow traffic from the vnet into them. As your workloads, system security agents, etc are deployed, consider adding even more NSG rules that help define the type of traffic that should and should not be traversing those subnet boundaries. Because each nodepool lives in its own subnet, you can apply more specific rules based on known/expected traffic patterns of your workload. diff --git a/docs/deploy/04-subscription.md b/docs/deploy/04-subscription.md index 93fa54c2..213cfbc5 100644 --- a/docs/deploy/04-subscription.md +++ b/docs/deploy/04-subscription.md @@ -17,8 +17,9 @@ The following three resource groups will be created in the steps below. | rg-enterprise-networking-hubs | Contains all of your organization's regional hubs. A regional hub resources in this implementation include an the hub Virtual Network, egress firewall, Azure Bastion, and Log Analytics for network logging. They may also contain your VPN Gateways, which are not addressed in this implementation. | | rg-enterprise-networking-spokes | Contains all of your organization's regional spokes and related networking resources. All spokes will peer with their regional hub and subnets will egress through the regional firewall in the hub. | | rg-bu0001a0005 | Contains the regulated cluster resources. | +| networkWatcherRG | Contains regional Network Watchers. _(This is only created if your subscription doesn't already have Network Watchers in place.)_ | -Both Azure Kubernetes Service and Azure Image Builder Service use a concept of a dynamically-created _infrastructure_ resource group. So in addition to the three resource groups mentioned above, as you follow these instructions, you'll end up with five resource groups; two of which are automatically created and their lifecycle tied to their owning service. You will not see these two infrastructure resource groups get created until later in the walkthrough when their owning service is created. +Both Azure Kubernetes Service and Azure Image Builder Service use a concept of a dynamically-created _infrastructure_ resource group. So in addition to the four resource groups mentioned above, as you follow these instructions, you'll end up with six resource groups; two of which are automatically created and their lifecycle tied to their owning service. You will not see these two infrastructure resource groups get created until later in the walkthrough when their owning service is created. ### Azure Policy applied @@ -27,9 +28,13 @@ To help govern our resources, there are policies we apply over the scope of thes | Policy Name | Scope | Purpose | |--------------------------------|---------------------------------|---------------------------------------------------------------------------------------------------| | Enable Azure Defender Standard | Subscription | Ensures that Azure Defender for Kubernetes, Container Service, and Key Vault are always enabled. | +| Deploy Network Watcher | Subscription | A _Deploy if not exists_ policy to ensure there is a regional network watcher for your virtual networks. _(This is only created if your subscription doesn't already have Network Watchers in place.)_ | | Allowed resource types | rg-enterprise-networking-hubs | Restricts the hub resource group to just relevant networking resources. | +| VNet must have Network Watcher | rg-enterprise-networking-hubs | Audit policy that will trigger if a network is deployed to a region that doesn't have a Network Watcher. _(This is only created if your subscription doesn't already have Network Watchers in place.)_ | | Allowed resource types | rg-enterprise-networking-spokes | Restricts the spokes resource group to just relevant networking resources. | +| VNet must have Network Watcher | rg-enterprise-networking-spokes | Audit policy that will trigger if a network is deployed to a region that doesn't have a Network Watcher. _(This is only created if your subscription doesn't already have Network Watchers in place.)_ | | Allowed resource types | rg-bu0001a0005 | Restricts the workload resource group to just resources necessary for this specific architecture. | +| Allowed resource types | networkWatcherRG | Restricts the Network Watcher resource group to just Network Watcher resources. _(This is only created if your subscription doesn't already have Network Watchers in place.)_ | | No public AKS clusters | rg-bu0001a0005 | Restricts the creation of AKS clusters to only those with private Kubernetes API server. | | No out-of-date AKS clusters | rg-bu0001a0005 | Restricts the creation of AKS clusters to only recent versions. | | No AKS clusters without RBAC | rg-bu0001a0005 | Restricts the creation of AKS clusters to only those that are Azure AD RBAC enabled. | @@ -76,20 +81,28 @@ Not only do we enable them in the steps below by default, but also set up an Azu TENANTID_AZURERBAC=$(az account show --query tenantId -o tsv) ``` +1. Check for existing Network Watchers. + + ```bash + [ $(az network watcher list --query 'length([])' -o tsv) -eq 0 ] && ENABLE_NETWORK_WATCHERS=true || ENABLE_NETWORK_WATCHERS=false + ``` + + > Azure Network Watchers are regional singletons in your subscription and should always be handled external to any specific workload; at the subscription level. Because your subscription may already have configuration (existing instances and/or a corresponding _deploy if not exists_ policy applied) around Network Watcher, it's hard to deliver a "one size fits all" solution in this isolated walkthrough. If the subscription you're deploying into doesn't have _any_ Network Watchers deployed, you'll have them set up as part of this deployment. If however, you already have Network Watcher resources, we'll leave those alone and won't deploy any additional resources related to them. If you have any conflicts in this walkthrough related to Network Watchers (existing management group policies, etc.), simply set `ENABLE_NETWORK_WATCHERS=false` and all related attempts to set up Network Watchers will be skipped. + 1. Perform subscription-level deployment. This will deploy the resource groups, Azure Policies, and Azure Security Center configuration all as identified above. ```bash # [This may take up to six minutes to run.] - az deployment sub create -f subscription.json -l centralus + az deployment sub create -f subscription.json -l centralus -p enableNetworkWatchers=${ENABLE_NETWORK_WATCHERS} ``` If you do not have permissions on your subscription to enable Azure Defender (which requires the Azure RBAC role of _Subscription Owner_ or _Security Admin_), then instead execute the following variation of the same command. This will not enable Azure Defender services nor will Azure Policy attempt to enable the same (the policy will still be created, but in audit-only mode). Your final implementation should be to a subscription with these security services activated. ```bash # [This may take up to five minutes to run.] - az deployment sub create -f subscription.json -l centralus -p enableAzureDefender=false enforceAzureDefenderAutoDeployPolicies=false + az deployment sub create -f subscription.json -l centralus -p enableAzureDefender=false enforceAzureDefenderAutoDeployPolicies=false enableNetworkWatchers=${ENABLE_NETWORK_WATCHERS} ``` ## Azure Security Benchmark diff --git a/docs/deploy/14-cleanup.md b/docs/deploy/14-cleanup.md index 27d9b401..9faac11a 100644 --- a/docs/deploy/14-cleanup.md +++ b/docs/deploy/14-cleanup.md @@ -16,6 +16,8 @@ After you are done exploring your deployed [AKS Baseline Cluster for Regulated W az group delete -n rg-enterprise-networking-hubs ``` + Depending on your subscription's starting point, this walkthrough might have also deployed a resource group by the name of `networkWatcherRG`. If you know this to be the case, and wish to remove it as well, you can execute `az group delete -n networkWatcherRG`. If you are not sure, you can leave that resource group in place, the resources deployed as part of this walkthrough to that resource group are not cost or security impacting. + 1. Purge Azure Key Vault > Because this reference implementation enables soft delete on Key Vault, execute a purge so your next deployment of this implementation doesn't run into a naming conflict. @@ -26,7 +28,7 @@ After you are done exploring your deployed [AKS Baseline Cluster for Regulated W 1. If any temporary changes were made to Azure AD or Azure RBAC permissions consider removing those as well. -1. [Remove the Azure Policy assignments](https://portal.azure.com/#blade/Microsoft_Azure_Policy/PolicyMenuBlade/Compliance) scoped to the cluster's resource group. To identify those created by this implementation, look for ones that are prefixed with `[your-cluster-name` and `[your-resource-group-names] `. If you added **Azure Security Benchmark** or **Enable Azure Defender Standard** as part of this as well, you can remove that. +1. [Remove the Azure Policy assignments](https://portal.azure.com/#blade/Microsoft_Azure_Policy/PolicyMenuBlade/Compliance) scoped to the cluster's resource group. To identify those created by this implementation, look for ones that are prefixed with `[your-cluster-name] ` and `[your-resource-group-names] `. If you added **Azure Security Benchmark** or **Enable Azure Defender Standard** as part of this as well, you may choose to remove them as well. Execute the following commands will handle all Resource Group-scoped policies: diff --git a/subscription.json b/subscription.json index 72547bc8..732db9ae 100644 --- a/subscription.json +++ b/subscription.json @@ -13,6 +13,12 @@ Notes: If unable to obtain permissions, you must pass false to the enableAzureDefender parameter or have someone else enable these for you. */ "parameters": { + "enableNetworkWatchers": { + "type": "bool", + "metadata": { + "description": "Set up Azure Policy to ensure Azure Network Watchers are deployed when vnets are created. They will be deployed to networkWatcherRG." + } + }, "enforceAzureDefenderAutoDeployPolicies": { "type": "bool", "defaultValue": true, @@ -31,6 +37,7 @@ "variables": { "rgName-Hubs": "rg-enterprise-networking-hubs", "rgName-Spokes": "rg-enterprise-networking-spokes", + "rgName-NetworkWatchers": "networkWatcherRG", // Name defined as part of the built-in Azure Policy a9b99dd8-06c5-4317-8629-9d86a3c6e7d9 "rgName-bu0001A0005": "rg-bu0001a0005", "policyDefinitionId-AllowedResources": "/providers/Microsoft.Authorization/policyDefinitions/a08ec900-254a-4555-9bf5-e42af04b5c5c", "policyDefinitionId-DenyAksWithoutPolicy": "/providers/Microsoft.Authorization/policyDefinitions/0a15ec92-a229-4763-bb14-0ea34a568f8d", @@ -38,6 +45,8 @@ "policyDefinitionId-DenyOldAks": "/providers/Microsoft.Authorization/policyDefinitions/fb893a29-21bb-418c-a157-e99480ec364c", "policyDefinitionId-CustomerManagedEncryption": "/providers/Microsoft.Authorization/policyDefinitions/7d7be79c-23ba-4033-84dd-45e2a5ccdd67", "policyDefinitionId-EncryptionAtHost": "/providers/Microsoft.Authorization/policyDefinitions/41425d9f-d1a5-499a-9932-f8ed8453932c", + "policyDefinitionId-DeployNetworkWatchers": "/providers/Microsoft.Authorization/policyDefinitions/a9b99dd8-06c5-4317-8629-9d86a3c6e7d9", + "policyDefinitionId-NetworkWatcherShouldBeEnabled": "/providers/Microsoft.Authorization/policyDefinitions/b6e2945c-0b7b-40f5-9233-7a5323b5cdc6", "policyDefinitionName-EnableAksDefender": "[guid(subscription().id, 'EnableDefenderForAks')]", "policyDefinitionName-EnableAcrDefender": "[guid(subscription().id, 'EnableDefenderForAcr')]", @@ -51,8 +60,10 @@ "policyDefinitionName-EncryptionAtHost": "[guid(subscription().id, 'EncryptionAtHost')]", "policyDefinitionName-NoPublicIPsForNICsInVnet": "[guid(subscription().id, 'NoPublicIPsForNICsInVnet')]", "policyDefinitionName-NoPublicIPsForVMScaleSets": "[guid(subscription().id, 'NoPublicIPsForVMScaleSets')]", + "policyAssignmentName-DeployNetworkWatchers": "[guid(subscription().id, variables('policyDefinitionId-DeployNetworkWatchers'))]", "policySetDefinitionName-EnableDefender": "[guid(subscription().id, 'EnableDefender')]", "roleId-SecurityAdmin": "/providers/Microsoft.Authorization/roleDefinitions/fb1c8493-542b-48eb-b624-b4c8fea62acd", + "roleId-NetworkContributor": "/providers/microsoft.authorization/roleDefinitions/4d97b98b-1d4f-4787-a291-c67834d212e7", "deploymentResourceRegion": "centralus" // This region is used as the default for all generic resource groups and for any additional deployment resources. No resources are actually deployed to this resource group. }, "resources": [ @@ -77,6 +88,14 @@ "location": "[variables('deploymentResourceRegion')]", "comments": "This is the resource group for BU001A0005. Typically this would be found in your workload's subscription." }, + { + "condition": "[parameters('enableNetworkWatchers')]", + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2020-10-01", + "name": "[variables('rgName-NetworkWatchers')]", + "location": "[variables('deploymentResourceRegion')]", + "comments": "This is the resource group for Azure Network Watchers. These are singletons per region and this RG will only be deployed if no existing Network Watchers are found in your subscription." + }, { "type": "Microsoft.Authorization/policyDefinitions", "apiVersion": "2019-09-01", @@ -495,6 +514,18 @@ } } } + }, + { + "condition": "[parameters('enableNetworkWatchers')]", + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2020-09-01", + "name": "[guid(variables('policyDefinitionId-NetworkWatcherShouldBeEnabled'), variables('rgName-Hubs'))]", + "comments": "Applying the 'Network Watcher Should be Enabled' policy to the Spoke resource group.", + "properties": { + "displayName": "[trim(take(concat('[', variables('rgName-Hubs'), '] ', reference(variables('policyDefinitionId-NetworkWatcherShouldBeEnabled'), '2020-09-01').displayName), 125))]", + "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('rgName-Hubs'))]", + "policyDefinitionId": "[variables('policyDefinitionId-NetworkWatcherShouldBeEnabled')]" + } } ] } @@ -539,6 +570,68 @@ } } } + }, + { + "condition": "[parameters('enableNetworkWatchers')]", + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2020-09-01", + "name": "[guid(variables('policyDefinitionId-NetworkWatcherShouldBeEnabled'), variables('rgName-Spokes'))]", + "comments": "Applying the 'Network Watcher Should be Enabled' policy to the Spoke resource group.", + "properties": { + "displayName": "[trim(take(concat('[', variables('rgName-Spokes'), '] ', reference(variables('policyDefinitionId-NetworkWatcherShouldBeEnabled'), '2020-09-01').displayName), 125))]", + "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('rgName-Spokes'))]", + "policyDefinitionId": "[variables('policyDefinitionId-NetworkWatcherShouldBeEnabled')]" + } + } + ] + } + } + }, + { + "condition": "[parameters('enableNetworkWatchers')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "[concat('Apply-', variables('rgName-NetworkWatchers'), '-Policies')]", + "resourceGroup": "[variables('rgName-NetworkWatchers')]", + "dependsOn": [ + "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('rgName-NetworkWatchers'))]", + "[subscriptionResourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentName-DeployNetworkWatchers'))]" + ], + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2020-09-01", + "name": "[guid(variables('policyDefinitionId-AllowedResources'), variables('rgName-NetworkWatchers'))]", + "comments": "Allowed Resources Policy applied to the network watchers resource group to only allow select networking resources.", + "properties": { + "displayName": "[trim(take(concat('[', variables('rgName-NetworkWatchers'), '] ', reference(variables('policyDefinitionId-AllowedResources'), '2020-09-01').displayName), 125))]", + "description": "List of supported resources for our Network Watcher resource group", + "scope": "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('rgName-NetworkWatchers'))]", + "policyDefinitionId": "[variables('policyDefinitionId-AllowedResources')]", + "parameters": { + "listOfResourceTypesAllowed": { + "value": [ + "Microsoft.Network/networkWatchers" + ] + } + } + } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2021-04-01-preview", + "name": "[guid(subscriptionResourceId('Microsoft.Authorization/policyAssignments', guid(variables('policyAssignmentName-DeployNetworkWatchers'))), variables('roleId-NetworkContributor'))]", + "properties": { + "roleDefinitionId": "[variables('roleId-NetworkContributor')]", + "description": "[concat('[Policy Managed Identity] ', 'Allows DINE for Network Watcher to deploy missing Network Watchers after VNets are deployed.')]", + "principalId": "[reference(subscriptionResourceId('Microsoft.Authorization/policyAssignments', variables('policyAssignmentName-DeployNetworkWatchers')), '2020-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } } ] } @@ -697,8 +790,29 @@ } }, { + "condition": "[parameters('enableNetworkWatchers')]", "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2019-09-01", + "apiVersion": "2020-09-01", + "name": "[variables('policyAssignmentName-DeployNetworkWatchers')]", + "location": "[variables('deploymentResourceRegion')]", + "dependsOn": [ + "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('rgName-NetworkWatchers'))]" + ], + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "displayName": "[reference(variables('policyDefinitionId-DeployNetworkWatchers'), '2020-09-01').displayName]", + "description": "[concat('Ensures that when a virtual network is created anywhere in the subscription, a Network Watcher for that region also exists. The policy requires that the Network Watchers be deployed to a resource group with the name ', variables('rgName-NetworkWatchers'), '.')]", + "enforcementMode": "Default", + "notScopes": [], + "scope": "[subscription().id]", + "policyDefinitionId": "[variables('policyDefinitionId-DeployNetworkWatchers')]" + } + }, + { + "type": "Microsoft.Authorization/policyAssignments", + "apiVersion": "2020-09-01", "name": "[guid(variables('policySetDefinitionName-EnableDefender'), subscription().id)]", "location": "[variables('deploymentResourceRegion')]", "dependsOn": [