Add ability to subscribe to DDoS detected alarms (#386)

* add alarm_notification_email field to CDNDedicatedWAFServiceInstance model

* add migration to create alarm_notification_email column

* update documentation URL

* add logic to set alarm_notification_email on instance from params for provision and update requests

* update logic for parsing alarm notification email

* add test for setting alarm_notification_email on API provision request

* update provision logic to raise error if CDN with dedicatd WAF instance does not specify alarm notification email

* update CDN provision tests

* refactor tests

* add tests for setting alarm notification email on API update request for CDN instances

* update migration for alarm_notification_email field to set field as non-nullable

* make alarm_notification_email field non-nullable

* add condition to return error if alarm_notification_email is unset when going from CDN to CDN dedicated WAF plan

* revert change to make alarm_notification_email non-nullable

* revert change to make alarm_notification_email non-nullable

* refactor logic for updating CDN to CDN dedicated WAF instance

* update API tests for plan update from CDN to CDN dedicated WAF instance

* update CDN to CDN dedicated WAF pipeline tests to properly set alarm_notification_email

* update tests for CDN dedicated WAF provisin pipeline to set alarm_notification_email

* refactor update logic

* add sns client

* add sns_notification_topic_arn field

* stub out logic for task to creatE a DDosDetected alarm

* add field for ddos_detected_cloudwatch_alarm_arn

* combine config fields into single field

* update test helper to set alarm_notification_email on CDN plan update

* refactor logic for create cloudwatch alarms

* change ddos_detected_cloudwatch_alarm_arn to ddos_detected_cloudwatch_alarm_name

* add test stubber for SNS

* create basic test for create_ddos_detected_alarm

* add test for creating ddos detection alarm

* add test for creating ddos alarm with tags

* create separate task for create_notification_topic

* add tests for create_notification_topic

* raise error if there is no SNS topic ARN when creating DDoS detected alarm & add test

* add delete_ddos_detected_alarm task and tests

* refactor cloudwatch health check alarms to use user provided notification email

* remove NOTIFICATIONS_SNS_TOPIC_ARN config

* remove test for NOTIFICATIONS_SNS_TOPIC_ARN

* add more tests for case of no topic provided

* add delete_notification_topic task and tests

* add tasks to pipelines

* update CDN dedicated WAF provision pipeline tests for creating new DDoS alarms

* update CDN dedicated WAF deprovision pipeline happy path

* update delete_notification_topic task to handle missing topic

* update tests for CDN dedicated WAF deprovision pipeline when resources are missing

* update pipeline tests for updating CDN to CDN dedicated WAF instance

* update pipeline tests for CDN dedicated WAF to specify alarm notification email

* update pipeline test alarm notification email

* fix provision common checks test

* fix update logic & update integration tests

* remove success notifications from pipeline

acceptance/run.sh

@@ -154,6 +154,13 @@ tests() {
       "$PLAN_NAME" \
       "$INSTANCE" \
       -c "{\"domains\": \"$DOMAIN_0, $DOMAIN_1\", \"forward_cookies\": \"cookieone, cookietwo\", \"forward_headers\": \"x-one-header,x-two-header\", \"error_responses\": {\"404\": \"/errors/404.html\"}}"
+  elif [[ "${PLAN_NAME}" == "domain-with-cdn-dedicated-wafP" ]]; then
+    echo "Creating the service instance"
+    cf create-service \
+      "$SERVICE_NAME" \
+      "$PLAN_NAME" \
+      "$INSTANCE" \
+      -c "{\"domains\": \"$DOMAIN_0, $DOMAIN_1\", \"alarm_notification_email\": \"$ALARM_NOTIFICATION_EMAIL\", \"forward_cookies\": \"cookieone, cookietwo\", \"forward_headers\": \"x-one-header,x-two-header\", \"error_responses\": {\"404\": \"/errors/404.html\"}}"
   else
     echo "Creating the service instance"
     cf create-service \

broker/api.py

@@ -25,7 +25,7 @@
 
 from broker import validators
 from broker.extensions import config, db
-from broker.lib.cdn import is_cdn_instance
+from broker.lib.cdn import is_cdn_instance, is_cdn_dedicated_waf_instance
 from broker.lib.tags import generate_instance_tags
 from broker.models import (
     Operation,
@@ -68,6 +68,7 @@
     queue_all_migration_deprovision_tasks_for_operation,
 )
 from broker.lib.utils import (
+    parse_alarm_notification_email,
     parse_cookie_options,
     parse_header_options,
     normalize_header_list,
@@ -98,7 +99,7 @@ def catalog(self) -> Service:
                 imageUrl="TODO",
                 longDescription="Create a custom domain to your application with TLS and an optional CDN. This will provision a TLS certificate from Let's Encrypt, a free certificate provider.",
                 providerDisplayName="Cloud.gov",
-                documentationUrl="https://github.com/cloud-gov/external-domain-broker",
+                documentationUrl="https://cloud.gov/docs/services/external-domain-service/",
                 supportUrl="https://cloud.gov/support",
             ),
             plans=[
@@ -336,16 +337,17 @@ def update(  # noqa C901 # TODO: simplify this function
             elif details.plan_id == CDN_DEDICATED_WAF_PLAN_ID:
                 queue = queue_all_cdn_to_cdn_dedicated_waf_update_tasks_for_operation
 
-                # update and commit any changes to the instance before changing its type,
+                # commit any changes to the instance before changing its type,
                 # which will wipe out any pending changes on `instance`
-                instance = update_cdn_instance(params, instance)
                 db.session.add(instance)
                 db.session.commit()
 
                 instance = change_instance_type(
                     instance, CDNDedicatedWAFServiceInstance, db.session
                 )
-                db.session.refresh(instance)
+                instance = update_cdn_instance(params, instance)
+                db.session.add(instance)
+                db.session.commit()
             else:
                 raise ClientError("Updating service plan is not supported")
         elif instance.instance_type == ServiceInstanceTypes.CDN_DEDICATED_WAF.value:
@@ -471,6 +473,15 @@ def provision_cdn_instance(
         instance.origin_protocol_policy = instance_type_model.ProtocolPolicy.HTTP.value
     else:
         instance.origin_protocol_policy = instance_type_model.ProtocolPolicy.HTTPS.value
+
+    alarm_notification_email = parse_alarm_notification_email(instance, params)
+    if alarm_notification_email:
+        instance.alarm_notification_email = alarm_notification_email
+    elif is_cdn_dedicated_waf_instance(instance) and not alarm_notification_email:
+        raise errors.ErrBadRequest(
+            f"'alarm_notification_email' is required for {ServiceInstanceTypes.CDN_DEDICATED_WAF.value} instances"
+        )
+
     return instance
 
 
@@ -520,6 +531,17 @@ def update_cdn_instance(params, instance):
         instance.error_responses = params["error_responses"]
         validators.ErrorResponseConfig(instance.error_responses).validate()
 
+    alarm_notification_email = parse_alarm_notification_email(instance, params)
+    if alarm_notification_email:
+        instance.alarm_notification_email = alarm_notification_email
+    elif (
+        is_cdn_dedicated_waf_instance(instance)
+        and not instance.alarm_notification_email
+    ):
+        raise errors.ErrBadRequest(
+            f"'alarm_notification_email' is required for {ServiceInstanceTypes.CDN_DEDICATED_WAF.value}"
+        )
+
     return instance
 
 

broker/aws.py

@@ -21,6 +21,7 @@
 )
 wafv2 = commercial_global_session.client("wafv2")
 cloudwatch_commercial = commercial_global_session.client("cloudwatch")
+sns_commercial = commercial_global_session.client("sns")
 
 govcloud_session = boto3.Session(
     region_name=config.AWS_GOVCLOUD_REGION,

broker/config.py

@@ -56,8 +56,7 @@ def __init__(self):
         # the maximum from AWS is 25, and we alert when we have 20 on a given alb
         self.MAX_CERTS_PER_ALB = 19
 
-        self.DEDICATED_WAF_NAME_PREFIX = f"cg-external-domains-{self.FLASK_ENV}"
-        self.CLOUDWATCH_ALARM_NAME_PREFIX = f"cg-external-domains-{self.FLASK_ENV}"
+        self.AWS_RESOURCE_PREFIX = f"cg-external-domains-{self.FLASK_ENV}"
 
         # see https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
         self.REQUEST_TIMEOUT = 30
@@ -134,8 +133,6 @@ def __init__(self):
         self.UAA_CLIENT_ID = self.env("UAA_CLIENT_ID")
         self.UAA_CLIENT_SECRET = self.env("UAA_CLIENT_SECRET")
 
-        self.NOTIFICATIONS_SNS_TOPIC_ARN = self.env("NOTIFICATIONS_SNS_TOPIC_ARN")
-
 
 class ProductionConfig(AppConfig):
     def __init__(self):
@@ -185,7 +182,6 @@ def __init__(self):
         self.ALB_LISTENER_ARNS = []
         self.DEDICATED_ALB_LISTENER_ARNS = []
         self.WAF_RATE_LIMIT_RULE_GROUP_ARN = "NONE"
-        self.NOTIFICATIONS_SNS_TOPIC_ARN = "NONE"
 
 
 class CheckDuplicateCertsConfig(UpgradeSchemaConfig):
@@ -262,8 +258,6 @@ def __init__(self):
         self.UAA_CLIENT_ID = "EXAMPLE"
         self.UAA_CLIENT_SECRET = "example"
 
-        self.NOTIFICATIONS_SNS_TOPIC_ARN = "fake-notifications-arn"
-
 
 class LocalDevelopmentConfig(DockerConfig):
     def __init__(self):

broker/lib/cdn.py

@@ -1,8 +1,14 @@
 from broker.models import ServiceInstanceTypes
 
 
-def is_cdn_instance(service_instance):
+def is_cdn_instance(service_instance) -> bool:
     return service_instance.instance_type in [
         ServiceInstanceTypes.CDN.value,
         ServiceInstanceTypes.CDN_DEDICATED_WAF.value,
     ]
+
+
+def is_cdn_dedicated_waf_instance(service_instance) -> bool:
+    return (
+        service_instance.instance_type == ServiceInstanceTypes.CDN_DEDICATED_WAF.value
+    )

broker/lib/utils.py

@@ -1,9 +1,9 @@
 import logging
 
 from broker import validators
-from broker.extensions import db
-from broker.lib.cdn import is_cdn_instance
+from broker.lib.cdn import is_cdn_dedicated_waf_instance
 from broker.models import (
+    ServiceInstanceTypes,
     CDNServiceInstance,
 )
 
@@ -56,6 +56,13 @@ def parse_domain_options(params) -> list[str]:
         return [d.strip().lower() for d in domains]
 
 
+def parse_alarm_notification_email(instance, params):
+    if not is_cdn_dedicated_waf_instance(instance):
+        return None
+
+    return params.get("alarm_notification_email")
+
+
 def validate_domain_name_changes(requested_domain_names, instance) -> list[str]:
     if len(requested_domain_names) > 0:
         logger.info("validating CNAMEs")

broker/models.py

@@ -192,6 +192,9 @@ class CDNDedicatedWAFServiceInstance(CDNServiceInstance):
     route53_health_checks = mapped_column(postgresql.JSONB, default=[])
     shield_associated_health_check = mapped_column(postgresql.JSONB, default={})
     cloudwatch_health_check_alarms = mapped_column(postgresql.JSONB, default=[])
+    alarm_notification_email = mapped_column(db.String)
+    sns_notification_topic_arn = mapped_column(db.String)
+    ddos_detected_cloudwatch_alarm_name = mapped_column(db.String)
 
     __mapper_args__ = {
         "polymorphic_identity": ServiceInstanceTypes.CDN_DEDICATED_WAF.value

broker/pipelines/cdn_dedicated_waf.py

@@ -9,6 +9,7 @@
     waf,
     shield,
     cloudwatch,
+    sns,
 )
 from broker.tasks.huey import huey
 
@@ -37,9 +38,11 @@ def queue_all_cdn_dedicated_waf_provision_tasks_for_operation(
         .then(cloudfront.wait_for_distribution, operation_id, **correlation)
         .then(route53.create_ALIAS_records, operation_id, **correlation)
         .then(route53.wait_for_changes, operation_id, **correlation)
+        .then(sns.create_notification_topic, operation_id, **correlation)
         .then(route53.create_new_health_checks, operation_id, **correlation)
         .then(shield.associate_health_check, operation_id, **correlation)
         .then(cloudwatch.create_health_check_alarms, operation_id, **correlation)
+        .then(cloudwatch.create_ddos_detected_alarm, operation_id, **correlation)
         .then(update_operations.provision, operation_id, **correlation)
     )
     huey.enqueue(task_pipeline)
@@ -57,9 +60,11 @@ def queue_all_cdn_dedicated_waf_deprovision_tasks_for_operation(
         update_operations.cancel_pending_provisioning.s(operation_id, **correlation)
         .then(route53.remove_ALIAS_records, operation_id, **correlation)
         .then(route53.remove_TXT_records, operation_id, **correlation)
+        .then(cloudwatch.delete_ddos_detected_alarm, operation_id, **correlation)
         .then(cloudwatch.delete_health_check_alarms, operation_id, **correlation)
         .then(shield.disassociate_health_check, operation_id, **correlation)
         .then(route53.delete_health_checks, operation_id, **correlation)
+        .then(sns.delete_notification_topic, operation_id, **correlation)
         .then(cloudfront.disable_distribution, operation_id, **correlation)
         .then(cloudfront.wait_for_distribution_disabled, operation_id, **correlation)
         .then(cloudfront.delete_distribution, operation_id=operation_id, **correlation)

broker/pipelines/plan_updates.py

@@ -8,6 +8,7 @@
     waf,
     shield,
     cloudwatch,
+    sns,
 )
 from broker.tasks.huey import huey
 
@@ -57,9 +58,11 @@ def queue_all_cdn_to_cdn_dedicated_waf_update_tasks_for_operation(
         .then(route53.create_ALIAS_records, operation_id, **correlation)
         .then(route53.wait_for_changes, operation_id, **correlation)
         .then(iam.delete_previous_server_certificate, operation_id, **correlation)
+        .then(sns.create_notification_topic, operation_id, **correlation)
         .then(route53.create_new_health_checks, operation_id, **correlation)
         .then(shield.associate_health_check, operation_id, **correlation)
         .then(cloudwatch.create_health_check_alarms, operation_id, **correlation)
+        .then(cloudwatch.create_ddos_detected_alarm, operation_id, **correlation)
         .then(update_operations.update_complete, operation_id, **correlation)
     )
     huey.enqueue(task_pipeline)