diff --git a/.github/workflows/deploy-aeolus.yml b/.github/workflows/deploy-aeolus.yml new file mode 100644 index 0000000..f8856af --- /dev/null +++ b/.github/workflows/deploy-aeolus.yml @@ -0,0 +1,86 @@ +name: Deploy aeolus + +on: + push: + branches: + - versions/aeolus + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Configure aws credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_PROD }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Get envionmental values + uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + secret-ids: | + AEOLUS, openaq-env/aeolus + name-transformation: uppercase + parse-json-secrets: true + + - uses: actions/setup-node@v3 + with: + node-version: "18" + + - name: build pages + working-directory: ./pages + run: | + yarn install + yarn run deploy + + - name: Install CDK + run: | + npm install -g aws-cdk@2.92.0 + + - uses: actions/setup-python@v3 + with: + python-version: '3.11' + + - name: Deploy stack + env: + ENV: "aeolus" + PROJECT: "openaq" + DATABASE_READ_USER: ${{ env.AEOLUS_DATABASE_READ_USER }} + DATABASE_READ_PASSWORD: ${{ env.AEOLUS_DATABASE_READ_PASSWORD }} + DATABASE_WRITE_USER: ${{ env.AEOLUS_DATABASE_WRITE_USER }} + DATABASE_WRITE_PASSWORD: ${{ env.AEOLUS_DATABASE_WRITE_PASSWORD }} + DATABASE_DB: ${{ env.AEOLUS_DATABASE_DB }} + DATABASE_HOST: ${{ env.AEOLUS_DATABASE_HOST }} + DATABASE_PORT: ${{ env.AEOLUS_DATABASE_PORT }} + API_LAMBDA_MEMORY_SIZE: ${{ env.AEOLUS_API_LAMBDA_MEMORY_SIZE }} + + CDK_ACCOUNT: ${{ secrets.CDK_ACCOUNT }} + CDK_REGION: ${{ secrets.CDK_REGION }} + + VPC_ID: ${{ env.AEOLUS_VPC_ID }} + + RATE_LIMITING: True + RATE_AMOUNT: 10 + RATE_AMOUNT_KEY: 10 + RATE_TIME: 1 + USER_AGENT: ${{ env.AEOLUS_USER_AGENT }} + ORIGIN: ${{ env.AEOLUS_ORIGIN }} + REDIS_HOST: ${{ env.AEOLUS_REDIS_HOST }} + REDIS_PORT: ${{ env.AEOLUS_REDIS_PORT }} + REDIS_SECURITY_GROUP_ID: ${{ env.AEOLUS_REDIS_SECURITY_GROUP_ID }} + + EMAIL_SENDER: ${{ env.AEOLUS_EMAIL_SENDER }} + SMTP_EMAIL_HOST: ${{ env.AEOLUS_SMTP_EMAIL_HOST }} + SMTP_EMAIL_USER: ${{ env.AEOLUS_SMTP_EMAIL_USER }} + SMTP_EMAIL_PASSWORD: ${{ env.AEOLUS_SMTP_EMAIL_PASSWORD }} + + EXPLORER_API_KEY: ${{ env.AEOLUS_EXPLORER_API_KEY }} + + working-directory: ./cdk + run: | + pip install -r requirements.txt + cdk deploy openaq-api-aeolus --require-approval never diff --git a/cdk/app.py b/cdk/app.py index 2058e2b..7ef1342 100644 --- a/cdk/app.py +++ b/cdk/app.py @@ -18,9 +18,9 @@ sys.path.insert(1, p) from openaq_api.settings import settings as lambda_env -p = os.path.abspath("../cloudfront_logs") -sys.path.insert(1, p) -from cloudfront_logs.settings import settings as cloudfront_logs_lambda_env +#p = os.path.abspath("../cloudfront_logs") +#sys.path.insert(1, p) +#from cloudfront_logs.settings import settings as cloudfront_logs_lambda_env app = aws_cdk.App() @@ -31,17 +31,11 @@ f"openaq-api-{settings.ENV}", env_name=settings.ENV, lambda_env=lambda_env, - cloudfront_logs_lambda_env=cloudfront_logs_lambda_env, vpc_id=settings.VPC_ID, - hosted_zone_name=settings.HOSTED_ZONE_NAME, - hosted_zone_id=settings.HOSTED_ZONE_ID, + redis_security_group_id=settings.REDIS_SECURITY_GROUP_ID, + redis_port=settings.REDIS_PORT, api_lambda_timeout=settings.API_LAMBDA_TIMEOUT, api_lambda_memory_size=settings.API_LAMBDA_MEMORY_SIZE, - cf_logs_lambda_timeout=settings.CF_LOG_LAMBDA_TIMEOUT, - cf_logs_lambda_memory_size=settings.CF_LOGS_LAMBDA_MEMORY_SIZE, - domain_name=settings.DOMAIN_NAME, - cert_arn=settings.CERTIFICATE_ARN, - web_acl_id=settings.WEB_ACL_ID, env=env, ) diff --git a/cdk/settings.py b/cdk/settings.py index 08d79db..139e15b 100644 --- a/cdk/settings.py +++ b/cdk/settings.py @@ -18,19 +18,11 @@ class Settings(BaseSettings): ENV: str = "staging" PROJECT: str = "openaq" API_CACHE_TIMEOUT: int = 900 - ROLLUP_LAMBDA_TIMEOUT: int = 900 - ROLLUP_LAMBDA_MEMORY_SIZE: int = 1536 LOG_LEVEL: str = "INFO" - HOSTED_ZONE_ID: str | None = None - HOSTED_ZONE_NAME: str | None = None - WEB_ACL_ID: str | None = None - DOMAIN_NAME: str | None = None - LOG_BUCKET: str | None = None - CERTIFICATE_ARN: str | None = None + REDIS_PORT: int | None = 6379 + REDIS_SECURITY_GROUP_ID: str | None = None API_LAMBDA_MEMORY_SIZE: int = 1536 API_LAMBDA_TIMEOUT: int = 15 # lambda timeout in seconds - CF_LOGS_LAMBDA_MEMORY_SIZE: int = 1792 - CF_LOG_LAMBDA_TIMEOUT: int = 15 * 60 # lambda timeout in seconds model_config = SettingsConfigDict( extra="ignore", diff --git a/cdk/stacks/lambda_api_stack.py b/cdk/stacks/lambda_api_stack.py index dbe7060..823390d 100644 --- a/cdk/stacks/lambda_api_stack.py +++ b/cdk/stacks/lambda_api_stack.py @@ -17,11 +17,6 @@ CfnOutput, Fn, aws_s3_notifications, - aws_route53 as route53, - aws_route53_targets as targets, - aws_certificatemanager as acm, - aws_cloudfront_origins as origins, - aws_cloudfront as cloudfront, ) from pydantic_settings import BaseSettings @@ -48,17 +43,11 @@ def __init__( env: Environment, env_name: str, lambda_env: BaseSettings, - cloudfront_logs_lambda_env: BaseSettings, api_lambda_memory_size: int, api_lambda_timeout: int, vpc_id: Union[str, None], - cf_logs_lambda_memory_size: Union[int, None], - cf_logs_lambda_timeout: Union[int, None], - hosted_zone_name: Union[str, None], - hosted_zone_id: Union[str, None], - domain_name: Union[str, None], - cert_arn: Union[str, None], - web_acl_id: Union[str, None], + redis_port: int, + redis_security_group_id: str, **kwargs, ) -> None: """Lambda to handle api requests""" @@ -69,6 +58,7 @@ def __init__( lambda_sec_group = None else: vpc = aws_ec2.Vpc.from_lookup(self, f"{id}-vpc", vpc_id=vpc_id) + lambda_sec_group = aws_ec2.SecurityGroup( self, f"openaq-api-lambda-sec-group_{env_name}", @@ -76,42 +66,6 @@ def __init__( vpc=vpc, allow_all_outbound=True, ) - redis_sec_group = aws_ec2.SecurityGroup( - self, - f"redis-sec-group_{env_name}", - security_group_name=f"redis-sec-group_{env_name}", - vpc=vpc, - allow_all_outbound=True, - ) - private_subnets_ids = [ps.subnet_id for ps in vpc.private_subnets] - redis_subnet_group = aws_elasticache.CfnSubnetGroup( - scope=self, - id=f"redis_subnet_group_{env_name}", - subnet_ids=private_subnets_ids, - description="subnet group for redis", - cache_subnet_group_name=f"redis-subnetgroup-{env_name}", - ) - redis_sec_group.add_ingress_rule( - peer=lambda_sec_group, - description="Allow Redis connection", - connection=aws_ec2.Port.tcp(6379), - ) - redis_cluster = aws_elasticache.CfnReplicationGroup( - self, - id=f"openaq-api-redis-cluster-{env_name}", - replication_group_description=f"openaq-api-redis-cluster-{env_name}", - engine="redis", - cache_node_type="cache.t4g.small", - replicas_per_node_group=1, - num_node_groups=2, - automatic_failover_enabled=True, - auto_minor_version_upgrade=True, - cache_subnet_group_name=redis_subnet_group.cache_subnet_group_name, - security_group_ids=[redis_sec_group.security_group_id], - ) - redis_cluster.add_depends_on(redis_subnet_group) - lambda_env.REDIS_HOST = redis_cluster.attr_configuration_end_point_address - lambda_env.REDIS_PORT = redis_cluster.attr_configuration_end_point_port lambda_env = stringify_settings(lambda_env) @@ -157,6 +111,21 @@ def __init__( ) ) + print(f"{redis_security_group_id} - {redis_port}") + if redis_security_group_id: + redis_security_group = aws_ec2.SecurityGroup.from_lookup_by_id( + self, + "SG", + security_group_id = redis_security_group_id, + ) + print(f"{redis_security_group.security_group_id} {redis_port}") + redis_security_group.add_ingress_rule( + peer=lambda_sec_group, + description="Allow Redis connection", + connection=aws_ec2.Port.tcp(6379), + remote_rule=True, + ) + api = HttpApi( self, f"{id}-endpoint", @@ -185,165 +154,3 @@ def __init__( api_url = f"https://{api.http_api_id}.execute-api.{self.region}.amazonaws.com" # TODO setup origin header to prevent traffic to API gateway directly CfnOutput(self, "Endpoint", value=api_url) - - if ( - domain_name - and cert_arn - and web_acl_id - and hosted_zone_id - and hosted_zone_name - ): - hosted_zone = route53.HostedZone.from_hosted_zone_attributes( - self, - f"openaq-api-hosted-zone-{env_name}", - hosted_zone_id=hosted_zone_id, - zone_name=hosted_zone_name, - ) - - cert = acm.Certificate.from_certificate_arn( - self, f"openaq-api-cert-{env_name}", cert_arn - ) - - cache_policy = cloudfront.CachePolicy( - self, - f"openaq-api-cache-policy-{env_name}", - cache_policy_name=f"openaq-api-cache-policy-{env_name}", - default_ttl=Duration.seconds(60), - min_ttl=Duration.minutes(0), - max_ttl=Duration.days(7), - cookie_behavior=cloudfront.CacheCookieBehavior.none(), - header_behavior=cloudfront.CacheHeaderBehavior.allow_list( - "X-API-Key", "User-Agent" - ), - query_string_behavior=cloudfront.CacheQueryStringBehavior.all(), - enable_accept_encoding_gzip=True, - enable_accept_encoding_brotli=True, - ) - - log_bucket = aws_s3.Bucket( - self, - f"openaq-api-dist-log-{env_name}", - bucket_name=f"openaq-api-dist-log-{env_name}", - auto_delete_objects=False, - public_read_access=False, - removal_policy=RemovalPolicy.DESTROY, - object_ownership=aws_s3.ObjectOwnership.OBJECT_WRITER, - lifecycle_rules=[ - aws_s3.LifecycleRule( - id=f"openaq-api-d ist-log-lifecycle-rule-{env_name}", - enabled=True, - expiration=aws_cdk.Duration.days(7), - ) - ], - ) - - log_event_queue = aws_sqs.Queue( - self, - f"openaq-api-cf-log-event-queue-{env_name}", - visibility_timeout=Duration.seconds(cf_logs_lambda_timeout), - ) - - log_bucket.add_event_notification( - aws_s3.EventType.OBJECT_CREATED_PUT, - aws_s3_notifications.SqsDestination(log_event_queue), - ) - - cloudfront_access_log_group = aws_logs.LogGroup( - self, - f"openaq-api-{env_name}-cf-access-log", - retention=aws_logs.RetentionDays.ONE_YEAR, - ) - - cloudfront_access_log_group.add_stream( - f"openaq-api-{env_name}-cf-access-log-stream", - log_stream_name=f"openaq-api-{env_name}-cf-access-log-stream", - ) - - log_lambda = aws_lambda.Function( - self, - f"openaq-api-cloudfront-logs-{env_name}-lambda", - code=aws_lambda.Code.from_asset( - path="../cloudfront_logs", - exclude=[ - "venv", - "__pycache__", - "pytest_cache", - ], - ), - handler="cloudfront_logs.main.handler", - runtime=aws_lambda.Runtime.PYTHON_3_11, - allow_public_subnet=True, - memory_size=cf_logs_lambda_memory_size, - environment=stringify_settings(cloudfront_logs_lambda_env), - timeout=Duration.seconds(cf_logs_lambda_timeout), - layers=[ - create_dependencies_layer( - self, - f"{env_name}", - "cloudfront_logs", - Path("../cloudfront_logs/requirements.txt"), - ), - ], - ) - - log_lambda.add_event_source( - SqsEventSource( - log_event_queue, - batch_size=10, - max_batching_window=Duration.minutes(1), - report_batch_item_failures=True, - ) - ) - - log_bucket.grant_read(log_lambda) - - origin_url = Fn.select( - 2, Fn.split("/", api_url) - ) # required to split url into compatible format for dist - - dist_origin_request_policy = cloudfront.OriginRequestPolicy( - self, - f"OpenAQAPIOriginRequestPolicy-{env_name}", - origin_request_policy_name=f"OpenAQAPIOriginRequestPolicy_{env_name}", - header_behavior=cloudfront.OriginRequestHeaderBehavior.allow_list( - "X-API-Key", "User-Agent" - ), - query_string_behavior=cloudfront.OriginRequestQueryStringBehavior.all(), - ) - - dist = cloudfront.Distribution( - self, - f"openaq-api-dist-{env_name}", - default_behavior=cloudfront.BehaviorOptions( - origin=origins.HttpOrigin(origin_url), - allowed_methods=cloudfront.AllowedMethods.ALLOW_ALL, - compress=True, - cache_policy=cache_policy, - viewer_protocol_policy=cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS, - origin_request_policy=dist_origin_request_policy, - ), - domain_names=[domain_name], - certificate=cert, - web_acl_id=web_acl_id, - enable_logging=True, - log_bucket=log_bucket, - ) - - route53.ARecord( - self, - f"openaq-api-alias-record-{env_name}", - zone=hosted_zone, - record_name=domain_name, - target=route53.RecordTarget.from_alias(targets.CloudFrontTarget(dist)), - ) - - CfnOutput(self, "Dist", value=dist.distribution_domain_name) - else: - print( - f""" - Could not add domain: {domain_name} - cert: {cert_arn} - zone_id: {hosted_zone_id} - zone_name: {hosted_zone_name} - """ - ) diff --git a/openaq_api/openaq_api/settings.py b/openaq_api/openaq_api/settings.py index ab1ce03..3b9e6fd 100644 --- a/openaq_api/openaq_api/settings.py +++ b/openaq_api/openaq_api/settings.py @@ -36,7 +36,6 @@ class Settings(BaseSettings): ORIGIN: str | None = None EMAIL_SENDER: str | None = None - SMTP_EMAIL_HOST: str | None = None SMTP_EMAIL_USER: str | None = None SMTP_EMAIL_PASSWORD: str | None = None