From 73d3133568d38f53204a77ad2e9e06f7ab3ab96d Mon Sep 17 00:00:00 2001 From: cavis Date: Tue, 19 Sep 2023 11:48:34 -0600 Subject: [PATCH 1/9] Setup shared app redis --- spire/templates/root.yml | 26 +++++++ spire/templates/shared-app-redis.yml | 105 +++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 spire/templates/shared-app-redis.yml diff --git a/spire/templates/root.yml b/spire/templates/root.yml index 182a66137..8218cd9df 100644 --- a/spire/templates/root.yml +++ b/spire/templates/root.yml @@ -464,6 +464,32 @@ Resources: TemplateURL: !Sub ${TemplateUrlBase}/spire/templates/shared-memcached.yml TimeoutInMinutes: 40 + # Requires VPC and ECS ASG SG + SharedAppRedisStack: + Type: AWS::CloudFormation::Stack + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + Parameters: + EnvironmentType: !Ref EnvironmentType + RootStackId: !Ref AWS::StackId + RootStackName: !Ref AWS::StackName + SourceSecurityGroupId1: !GetAtt SharedEcsAsgSecurityGroupStack.Outputs.InstanceSecurityGroupId + VpcId: !GetAtt SharedVpcStack.Outputs.VpcId + VpcPrivateSubnet1Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet1Id + VpcPrivateSubnet2Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet2Id + VpcPrivateSubnet3Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet3Id + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref AWS::StackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:application, Value: Common } + TemplateURL: !Sub ${TemplateUrlBase}/spire/templates/shared-app-redis.yml + TimeoutInMinutes: 40 + # Requires VPC SharedRedisSecurityGroupStack: Type: AWS::CloudFormation::Stack diff --git a/spire/templates/shared-app-redis.yml b/spire/templates/shared-app-redis.yml new file mode 100644 index 000000000..17ec3d8eb --- /dev/null +++ b/spire/templates/shared-app-redis.yml @@ -0,0 +1,105 @@ +# stacks/shared-app-redis.yml +AWSTemplateFormatVersion: "2010-09-09" + +Description: >- + Creates a Redis cluster intended to be used by several applications. Unlike + the shared-redis/cluster.yml, this is a non-redundant single node redis. It + should only be used for pure FIFO caching, with no state we need to hold + onto if it should cycle or disappear. The cluster's security group will allow + only traffic from a security group provided as a stack parameter. + +Parameters: + EnvironmentType: { Type: String } + RootStackName: { Type: String } + RootStackId: { Type: String } + SourceSecurityGroupId1: { Type: AWS::EC2::SecurityGroup::Id } + StagingInstanceType: + Type: String + Default: cache.t4g.micro + ProductionInstanceType: + Type: String + Default: cache.t4g.small + VpcId: { Type: AWS::EC2::VPC::Id } + VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id } + VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id } + VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id } + +Conditions: + IsProduction: !Equals [!Ref EnvironmentType, Production] + +Resources: + SharedAppRedisSecurityGroup: + Type: AWS::EC2::SecurityGroup + Condition: IsPrimaryRegion + Properties: + GroupDescription: !Sub Shared ${EnvironmentType} App Redis security group + SecurityGroupIngress: + - FromPort: 6379 + IpProtocol: tcp + SourceSecurityGroupId: !Ref SourceSecurityGroupId1 + ToPort: 6379 + Tags: + - { Key: Name, Value: !Sub "${RootStackName}_shared_app_redis" } + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:application, Value: Common } + VpcId: !Ref VpcId + + SharedAppRedisSubnetGroup: + Type: AWS::ElastiCache::SubnetGroup + Condition: IsPrimaryRegion + Properties: + Description: !Sub Shared ${EnvironmentType} App Redis subnet group + SubnetIds: + - !Ref VpcPrivateSubnet1Id + - !Ref VpcPrivateSubnet2Id + - !Ref VpcPrivateSubnet3Id + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:application, Value: Common } + + SharedAppRedisCluster: + Type: AWS::ElastiCache::CacheCluster + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Condition: IsPrimaryRegion + Properties: + AutoMinorVersionUpgrade: true + CacheNodeType: !If + - IsProduction + - !Ref ProductionInstanceType + - !Ref StagingInstanceType + CacheSubnetGroupName: !Ref SharedAppRedisSubnetGroup + Engine: redis + EngineVersion: 6.x + NumCacheNodes: 1 + PreferredMaintenanceWindow: sun:09:00-sun:10:00 + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:application, Value: Common } + VpcSecurityGroupIds: + - !Ref SharedAppRedisSecurityGroup + +Outputs: + CacheName: + Value: !If [IsPrimaryRegion, !Ref SharedAppRedisCluster, !Ref "AWS::NoValue"] + CacheEndpointAddress: + Description: Cache endpoint hostname + Value: !If [IsPrimaryRegion, !GetAtt SharedAppRedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"] + CacheEndpointPort: + Description: Cache endpoint port + Value: !If [IsPrimaryRegion, !GetAtt SharedAppRedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"] From 1c49a69f63252a629d5a24568c736fb0d7590db8 Mon Sep 17 00:00:00 2001 From: cavis Date: Tue, 19 Sep 2023 11:55:49 -0600 Subject: [PATCH 2/9] Remove augury redis; use shared-app-redis instead --- spire/templates/apps-300A.yml | 11 ++--- spire/templates/apps/augury.yml | 82 ++++----------------------------- spire/templates/root.yml | 5 +- 3 files changed, 14 insertions(+), 84 deletions(-) diff --git a/spire/templates/apps-300A.yml b/spire/templates/apps-300A.yml index ec316dbb4..c3609ac64 100644 --- a/spire/templates/apps-300A.yml +++ b/spire/templates/apps-300A.yml @@ -20,9 +20,6 @@ Parameters: RootStackId: { Type: String } VpcId: { Type: AWS::EC2::VPC::Id } NewRelicApiKeyPrxLite: { Type: String } - VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id } - VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id } - VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id } VpcPublicSubnet1Id: { Type: AWS::EC2::Subnet::Id } VpcPublicSubnet2Id: { Type: AWS::EC2::Subnet::Id } VpcPublicSubnet3Id: { Type: AWS::EC2::Subnet::Id } @@ -34,6 +31,8 @@ Parameters: SharedEcsAsgInstanceSecurityGroupId: { Type: AWS::EC2::SecurityGroup::Id } SharedRedisReplicationGroupEndpointAddress: { Type: String } SharedRedisReplicationGroupEndpointPort: { Type: String } + SharedAppRedisEndpointAddress: { Type: String } + SharedAppRedisEndpointPort: { Type: String } SharedAuroraPostgresqlEndpoint: { Type: String } SharedAuroraPostgresqlPort: { Type: String } SharedPostgresqlClientSecurityGroupId: { Type: String } @@ -83,17 +82,15 @@ Resources: RootStackId: !Ref RootStackId EchoServiceToken: !Ref EchoServiceToken CloudWatchAlarmTaggerServiceToken: !Ref CloudWatchAlarmTaggerServiceToken - VpcPrivateSubnet1Id: !Ref VpcPrivateSubnet1Id - VpcPrivateSubnet2Id: !Ref VpcPrivateSubnet2Id - VpcPrivateSubnet3Id: !Ref VpcPrivateSubnet3Id VpcPublicSubnet1Id: !Ref VpcPublicSubnet1Id VpcPublicSubnet2Id: !Ref VpcPublicSubnet2Id VpcPublicSubnet3Id: !Ref VpcPublicSubnet3Id EcsLaunchEndpointsAccessSecurityGroupId: !Ref EcsLaunchEndpointsAccessSecurityGroupId KmsEndpointAccessSecurityGroupId: !Ref KmsEndpointAccessSecurityGroupId + SharedAppRedisEndpointAddress: !Ref SharedAppRedisEndpointAddress + SharedAppRedisEndpointPort: !Ref SharedAppRedisEndpointPort SharedAuroraPostgresqlEndpoint: !Ref SharedAuroraPostgresqlEndpoint SharedAuroraPostgresqlPort: !Ref SharedAuroraPostgresqlPort - SharedEcsAsgInstanceSecurityGroupId: !Ref SharedEcsAsgInstanceSecurityGroupId SharedPostgresqlClientSecurityGroupId: !Ref SharedPostgresqlClientSecurityGroupId CastlePostgresInstanceEndpointAddress: !Ref CastlePostgresInstanceEndpointAddress CastlePostgresInstanceEndpointPort: !Ref CastlePostgresInstanceEndpointPort diff --git a/spire/templates/apps/augury.yml b/spire/templates/apps/augury.yml index 1aff13c5a..6ba44655c 100644 --- a/spire/templates/apps/augury.yml +++ b/spire/templates/apps/augury.yml @@ -40,17 +40,15 @@ Parameters: NewRelicApiKeyPrxLite: { Type: String } EcrImageTag: { Type: AWS::SSM::Parameter::Value } AlbListenerRulePriorityPrefix: { Type: String } - VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id } - VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id } - VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id } VpcPublicSubnet1Id: { Type: AWS::EC2::Subnet::Id } VpcPublicSubnet2Id: { Type: AWS::EC2::Subnet::Id } VpcPublicSubnet3Id: { Type: AWS::EC2::Subnet::Id } EcsLaunchEndpointsAccessSecurityGroupId: { Type: AWS::EC2::SecurityGroup::Id } KmsEndpointAccessSecurityGroupId: { Type: AWS::EC2::SecurityGroup::Id } + SharedAppRedisEndpointAddress: { Type: String } + SharedAppRedisEndpointPort: { Type: String } SharedAuroraPostgresqlEndpoint: { Type: String } SharedAuroraPostgresqlPort: { Type: String } - SharedEcsAsgInstanceSecurityGroupId: { Type: String } SharedPostgresqlClientSecurityGroupId: { Type: String } CastlePostgresInstanceEndpointAddress: { Type: String } CastlePostgresInstanceEndpointPort: { Type: String } @@ -84,70 +82,6 @@ Resources: SlowWorkerLoggedErrorsMetricName: !Sub SlowWorkerLoggedErrors${EnvironmentType} WebLoggedErrorsMetricName: !Sub WebLoggedErrors${EnvironmentType} - RedisSecurityGroup: - Type: AWS::EC2::SecurityGroup - Condition: IsPrimaryRegion - Properties: - GroupDescription: !Sub Augury ${EnvironmentType} Redis security group - SecurityGroupIngress: - - FromPort: 6379 - IpProtocol: tcp - SourceSecurityGroupId: !Ref SharedEcsAsgInstanceSecurityGroupId - ToPort: 6379 - Tags: - - { Key: Name, Value: !Sub "${RootStackName}_augury_redis" } - - { Key: prx:meta:tagging-version, Value: "2021-04-07" } - - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } - - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } - - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } - - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } - - { Key: prx:ops:environment, Value: !Ref EnvironmentType } - - { Key: prx:dev:family, Value: Dovetail } - - { Key: prx:dev:application, Value: Augury } - VpcId: !Ref VpcId - RedisSubnetGroup: - Type: AWS::ElastiCache::SubnetGroup - Condition: IsPrimaryRegion - Properties: - Description: !Sub Augury ${EnvironmentType} Redis subnet group - SubnetIds: - - !Ref VpcPrivateSubnet1Id - - !Ref VpcPrivateSubnet2Id - - !Ref VpcPrivateSubnet3Id - Tags: - - { Key: prx:meta:tagging-version, Value: "2021-04-07" } - - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } - - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } - - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } - - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } - - { Key: prx:ops:environment, Value: !Ref EnvironmentType } - - { Key: prx:dev:family, Value: Dovetail } - - { Key: prx:dev:application, Value: Augury } - RedisCluster: - Type: AWS::ElastiCache::CacheCluster - DeletionPolicy: Delete - UpdateReplacePolicy: Delete - Condition: IsPrimaryRegion - Properties: - AutoMinorVersionUpgrade: false - CacheNodeType: !If [IsProduction, cache.t4g.small, cache.t4g.micro] - CacheSubnetGroupName: !Ref RedisSubnetGroup - Engine: redis - EngineVersion: 6.x - NumCacheNodes: 1 - PreferredMaintenanceWindow: sun:09:00-sun:10:00 - Tags: - - { Key: prx:meta:tagging-version, Value: "2021-04-07" } - - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } - - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } - - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } - - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } - - { Key: prx:ops:environment, Value: !Ref EnvironmentType } - - { Key: prx:dev:family, Value: Dovetail } - - { Key: prx:dev:application, Value: Augury } - VpcSecurityGroupIds: - - !Ref RedisSecurityGroup - HostHeaderListenerRule: Type: AWS::ElasticLoadBalancingV2::ListenerRule Properties: @@ -365,9 +299,9 @@ Resources: - Name: POSTGRES_PORT Value: !Ref SharedAuroraPostgresqlPort - Name: REDIS_HOST - Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"] + Value: !Ref SharedAppRedisEndpointAddress - Name: REDIS_PORT - Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"] + Value: !Ref SharedAppRedisEndpointPort - Name: UPLOAD_ADFILE_SECRET_ACCESS_KEY Value: !GetAtt S3SigningAccessKey.SecretAccessKey - Name: UPLOAD_ADFILE_ACCESS_KEY_ID @@ -504,9 +438,9 @@ Resources: - Name: POSTGRES_PORT Value: !Ref SharedAuroraPostgresqlPort - Name: REDIS_HOST - Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"] + Value: !Ref SharedAppRedisEndpointAddress - Name: REDIS_PORT - Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"] + Value: !Ref SharedAppRedisEndpointPort - Name: RAILS_SERVE_STATIC_FILES Value: "true" - Name: POSTGRES_POOL_SIZE @@ -998,9 +932,9 @@ Resources: - Name: POSTGRES_PORT Value: !Ref SharedAuroraPostgresqlPort - Name: REDIS_HOST - Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"] + Value: !Ref SharedAppRedisEndpointAddress - Name: REDIS_PORT - Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"] + Value: !Ref SharedAppRedisEndpointPort - Name: RAILS_SERVE_STATIC_FILES Value: "true" - Name: POSTGRES_POOL_SIZE diff --git a/spire/templates/root.yml b/spire/templates/root.yml index 8218cd9df..52f704300 100644 --- a/spire/templates/root.yml +++ b/spire/templates/root.yml @@ -733,9 +733,6 @@ Resources: RegionMode: !FindInMap [RegionModeMap, !Ref "AWS::Region", !Ref EnvironmentType] RootStackName: !Ref AWS::StackName RootStackId: !Ref AWS::StackId - VpcPrivateSubnet1Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet1Id - VpcPrivateSubnet2Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet2Id - VpcPrivateSubnet3Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet3Id VpcPublicSubnet1Id: !GetAtt SharedVpcStack.Outputs.PublicSubnet1Id VpcPublicSubnet2Id: !GetAtt SharedVpcStack.Outputs.PublicSubnet2Id VpcPublicSubnet3Id: !GetAtt SharedVpcStack.Outputs.PublicSubnet3Id @@ -747,6 +744,8 @@ Resources: SharedEcsAsgInstanceSecurityGroupId: !GetAtt SharedEcsAsgSecurityGroupStack.Outputs.InstanceSecurityGroupId SharedRedisReplicationGroupEndpointAddress: !GetAtt SharedRedisStack.Outputs.ReplicationGroupEndpointAddress SharedRedisReplicationGroupEndpointPort: !GetAtt SharedRedisStack.Outputs.ReplicationGroupEndpointPort + SharedAppRedisEndpointAddress: !GetAtt SharedAppRedisStack.Outputs.CacheEndpointAddress + SharedAppRedisEndpointPort: !GetAtt SharedAppRedisStack.Outputs.CacheEndpointPort SharedAuroraPostgresqlEndpoint: !Ref SharedAuroraPostgresqlEndpoint SharedAuroraPostgresqlPort: !Ref SharedAuroraPostgresqlPort SharedPostgresqlClientSecurityGroupId: !GetAtt SharedDatabaseSecurityGroupsStack.Outputs.SharedPostgresqlClientSecurityGroupId From 9cc9d28ffcfd58a0681871e4e97baafdbdfa52be Mon Sep 17 00:00:00 2001 From: cavis Date: Tue, 19 Sep 2023 11:59:55 -0600 Subject: [PATCH 3/9] Add redis to feeder ENVs --- spire/templates/apps-200A.yml | 4 ++++ spire/templates/apps/feeder.yml | 10 ++++++++++ spire/templates/root.yml | 2 ++ 3 files changed, 16 insertions(+) diff --git a/spire/templates/apps-200A.yml b/spire/templates/apps-200A.yml index f0dc4f700..0a78b36ce 100644 --- a/spire/templates/apps-200A.yml +++ b/spire/templates/apps-200A.yml @@ -29,6 +29,8 @@ Parameters: VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id } VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id } VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id } + SharedAppRedisEndpointAddress: { Type: String } + SharedAppRedisEndpointPort: { Type: String } SharedMemcachedEndpointAddress: { Type: String } SharedMemcachedEndpointPort: { Type: String } SharedRedisReplicationGroupEndpointAddress: { Type: String } @@ -274,6 +276,8 @@ Resources: S3SigningUserName: !Ref S3SigningUserName S3SigningEndpointUrl: !Ref S3SigningEndpointUrl S3SigningAccessKeyId: !Ref S3SigningAccessKeyId + SharedAppRedisEndpointAddress: !Ref SharedAppRedisEndpointAddress + SharedAppRedisEndpointPort: !Ref SharedAppRedisEndpointPort SharedAuroraPostgresqlEndpoint: !Ref SharedAuroraPostgresqlEndpoint SharedAuroraPostgresqlPort: !Ref SharedAuroraPostgresqlPort PorterJobExecutionSnsTopicArn: !Ref PorterJobExecutionSnsTopicArn diff --git a/spire/templates/apps/feeder.yml b/spire/templates/apps/feeder.yml index 3af22ef1b..7eeeda3ad 100644 --- a/spire/templates/apps/feeder.yml +++ b/spire/templates/apps/feeder.yml @@ -45,6 +45,8 @@ Parameters: NewRelicApiKeyPrxLite: { Type: String } EcrImageTag: { Type: AWS::SSM::Parameter::Value } AlbListenerRulePriorityPrefix: { Type: String } + SharedAppRedisEndpointAddress: { Type: String } + SharedAppRedisEndpointPort: { Type: String } SharedAuroraPostgresqlEndpoint: { Type: String } SharedAuroraPostgresqlPort: { Type: String } AnnounceResourcePrefix: { Type: String } @@ -673,6 +675,10 @@ Resources: Value: !GetAtt Constants.PublicFeedsUrlPrefix - Name: RAILS_ENV Value: !Ref EnvironmentTypeLowercase + - Name: REDIS_HOST + Value: !Ref SharedAppRedisEndpointAddress + - Name: REDIS_PORT + Value: !Ref SharedAppRedisEndpointPort - Name: UPLOAD_BUCKET_NAME Value: !Ref FeederUploadsBucket - Name: UPLOAD_S3_ENDPOINT_HOST @@ -813,6 +819,10 @@ Resources: Value: !GetAtt Constants.PublicFeedsUrlPrefix - Name: RAILS_ENV Value: !Ref EnvironmentTypeLowercase + - Name: REDIS_HOST + Value: !Ref SharedAppRedisEndpointAddress + - Name: REDIS_PORT + Value: !Ref SharedAppRedisEndpointPort - Name: START_SAY_WHEN Value: true - Name: WORKER_COUNT diff --git a/spire/templates/root.yml b/spire/templates/root.yml index 52f704300..e4eafb3c9 100644 --- a/spire/templates/root.yml +++ b/spire/templates/root.yml @@ -642,6 +642,8 @@ Resources: VpcPrivateSubnet1Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet1Id VpcPrivateSubnet2Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet2Id VpcPrivateSubnet3Id: !GetAtt SharedVpcStack.Outputs.PrivateSubnet3Id + SharedAppRedisEndpointAddress: !GetAtt SharedAppRedisStack.Outputs.CacheEndpointAddress + SharedAppRedisEndpointPort: !GetAtt SharedAppRedisStack.Outputs.CacheEndpointPort SharedMemcachedEndpointAddress: !GetAtt SharedMemcachedStack.Outputs.CacheEndpointAddress SharedMemcachedEndpointPort: !GetAtt SharedMemcachedStack.Outputs.CacheEndpointPort SharedRedisReplicationGroupEndpointAddress: !GetAtt SharedRedisStack.Outputs.ReplicationGroupEndpointAddress From d74d23d9e8743214561522d4050178265075fdf5 Mon Sep 17 00:00:00 2001 From: cavis Date: Tue, 19 Sep 2023 12:05:04 -0600 Subject: [PATCH 4/9] Fixup region mode --- spire/templates/apps-300A.yml | 1 - spire/templates/apps/augury.yml | 2 -- spire/templates/root.yml | 1 + spire/templates/shared-app-redis.yml | 10 ++++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/spire/templates/apps-300A.yml b/spire/templates/apps-300A.yml index c3609ac64..9fe5e80a0 100644 --- a/spire/templates/apps-300A.yml +++ b/spire/templates/apps-300A.yml @@ -77,7 +77,6 @@ Resources: EnvironmentType: !Ref EnvironmentType EnvironmentTypeAbbreviation: !Ref EnvironmentTypeAbbreviation EnvironmentTypeLowercase: !Ref EnvironmentTypeLowercase - RegionMode: !Ref RegionMode RootStackName: !Ref RootStackName RootStackId: !Ref RootStackId EchoServiceToken: !Ref EchoServiceToken diff --git a/spire/templates/apps/augury.yml b/spire/templates/apps/augury.yml index 6ba44655c..21ef717d5 100644 --- a/spire/templates/apps/augury.yml +++ b/spire/templates/apps/augury.yml @@ -31,7 +31,6 @@ Parameters: EnvironmentType: { Type: String } EnvironmentTypeAbbreviation: { Type: String } EnvironmentTypeLowercase: { Type: String } - RegionMode: { Type: String } RootStackName: { Type: String } RootStackId: { Type: String } EchoServiceToken: { Type: String } @@ -64,7 +63,6 @@ Parameters: Conditions: IsProduction: !Equals [!Ref EnvironmentType, Production] - IsPrimaryRegion: !Equals [!Ref RegionMode, Primary] HasAuroraEndpoint: !Not [!Equals [!Ref SharedAuroraPostgresqlEndpoint, ""]] EnableWorkers: !And [!Condition HasAuroraEndpoint, !Condition IsPrimaryRegion] diff --git a/spire/templates/root.yml b/spire/templates/root.yml index e4eafb3c9..89c792f6b 100644 --- a/spire/templates/root.yml +++ b/spire/templates/root.yml @@ -472,6 +472,7 @@ Resources: Properties: Parameters: EnvironmentType: !Ref EnvironmentType + RegionMode: !FindInMap [RegionModeMap, !Ref "AWS::Region", !Ref EnvironmentType] RootStackId: !Ref AWS::StackId RootStackName: !Ref AWS::StackName SourceSecurityGroupId1: !GetAtt SharedEcsAsgSecurityGroupStack.Outputs.InstanceSecurityGroupId diff --git a/spire/templates/shared-app-redis.yml b/spire/templates/shared-app-redis.yml index 17ec3d8eb..8570e29e8 100644 --- a/spire/templates/shared-app-redis.yml +++ b/spire/templates/shared-app-redis.yml @@ -10,21 +10,23 @@ Description: >- Parameters: EnvironmentType: { Type: String } + RegionMode: { Type: String } RootStackName: { Type: String } RootStackId: { Type: String } SourceSecurityGroupId1: { Type: AWS::EC2::SecurityGroup::Id } + VpcId: { Type: AWS::EC2::VPC::Id } + VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id } + VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id } + VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id } StagingInstanceType: Type: String Default: cache.t4g.micro ProductionInstanceType: Type: String Default: cache.t4g.small - VpcId: { Type: AWS::EC2::VPC::Id } - VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id } - VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id } - VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id } Conditions: + IsPrimaryRegion: !Equals [!Ref RegionMode, Primary] IsProduction: !Equals [!Ref EnvironmentType, Production] Resources: From 8deb69bda9e1a209fcfa9e47103e9f078d284caf Mon Sep 17 00:00:00 2001 From: cavis Date: Tue, 19 Sep 2023 12:14:35 -0600 Subject: [PATCH 5/9] Still need region mode --- spire/templates/apps-300A.yml | 1 + spire/templates/apps/augury.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/spire/templates/apps-300A.yml b/spire/templates/apps-300A.yml index 9fe5e80a0..c3609ac64 100644 --- a/spire/templates/apps-300A.yml +++ b/spire/templates/apps-300A.yml @@ -77,6 +77,7 @@ Resources: EnvironmentType: !Ref EnvironmentType EnvironmentTypeAbbreviation: !Ref EnvironmentTypeAbbreviation EnvironmentTypeLowercase: !Ref EnvironmentTypeLowercase + RegionMode: !Ref RegionMode RootStackName: !Ref RootStackName RootStackId: !Ref RootStackId EchoServiceToken: !Ref EchoServiceToken diff --git a/spire/templates/apps/augury.yml b/spire/templates/apps/augury.yml index 21ef717d5..6ba44655c 100644 --- a/spire/templates/apps/augury.yml +++ b/spire/templates/apps/augury.yml @@ -31,6 +31,7 @@ Parameters: EnvironmentType: { Type: String } EnvironmentTypeAbbreviation: { Type: String } EnvironmentTypeLowercase: { Type: String } + RegionMode: { Type: String } RootStackName: { Type: String } RootStackId: { Type: String } EchoServiceToken: { Type: String } @@ -63,6 +64,7 @@ Parameters: Conditions: IsProduction: !Equals [!Ref EnvironmentType, Production] + IsPrimaryRegion: !Equals [!Ref RegionMode, Primary] HasAuroraEndpoint: !Not [!Equals [!Ref SharedAuroraPostgresqlEndpoint, ""]] EnableWorkers: !And [!Condition HasAuroraEndpoint, !Condition IsPrimaryRegion] From 25536a2faea86b5e43ec14106d498317318f3965 Mon Sep 17 00:00:00 2001 From: cavis Date: Tue, 19 Sep 2023 13:01:29 -0600 Subject: [PATCH 6/9] Return blank strings, not noval ... applications using this should handle blanks --- spire/templates/shared-app-redis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spire/templates/shared-app-redis.yml b/spire/templates/shared-app-redis.yml index 8570e29e8..9d977114a 100644 --- a/spire/templates/shared-app-redis.yml +++ b/spire/templates/shared-app-redis.yml @@ -98,10 +98,10 @@ Resources: Outputs: CacheName: - Value: !If [IsPrimaryRegion, !Ref SharedAppRedisCluster, !Ref "AWS::NoValue"] + Value: !If [IsPrimaryRegion, !Ref SharedAppRedisCluster, ""] CacheEndpointAddress: Description: Cache endpoint hostname - Value: !If [IsPrimaryRegion, !GetAtt SharedAppRedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"] + Value: !If [IsPrimaryRegion, !GetAtt SharedAppRedisCluster.RedisEndpoint.Address, ""] CacheEndpointPort: Description: Cache endpoint port - Value: !If [IsPrimaryRegion, !GetAtt SharedAppRedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"] + Value: !If [IsPrimaryRegion, !GetAtt SharedAppRedisCluster.RedisEndpoint.Port, ""] From e28d92710dbcbd5074ee5ae80b004086db3a8d17 Mon Sep 17 00:00:00 2001 From: Chris Kalafarski Date: Tue, 19 Sep 2023 15:07:26 -0400 Subject: [PATCH 7/9] Add shared app redis to dashboard --- spire/templates/dashboards.yml | 5 +++-- spire/templates/root.yml | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/spire/templates/dashboards.yml b/spire/templates/dashboards.yml index 3c39811c4..de6706594 100644 --- a/spire/templates/dashboards.yml +++ b/spire/templates/dashboards.yml @@ -25,7 +25,8 @@ Parameters: SharedVpcFlowLogsLogGroupName: { Type: String } SharedMemcachedCacheName: { Type: String } - SharedRedisReplicationGroupName: { Type: String } + SharedAppRedisCacheName: { Type: String } + DovetailRedisReplicationGroupName: { Type: String } StackResourceGroupName: { Type: String } StackLogGroupsGroupName: { Type: String } @@ -97,7 +98,7 @@ Resources: "x": 0, "type": "text", "properties": { - "markdown": "\n# AWS Console\n\n- [CloudFormation root stack](https://console.aws.amazon.com/cloudformation/home?region=${AWS::Region}#/stacks/stackinfo?stackId=${RootStackId})\n- [Stack resource group](https://console.aws.amazon.com/resource-groups/group/${StackResourceGroupName}?region=${AWS::Region}) | [Logs](https://console.aws.amazon.com/resource-groups/group/${StackLogGroupsGroupName}?region=${AWS::Region})\n- [ECS Cluster](https://console.aws.amazon.com/ecs/home?region=${AWS::Region}#/clusters/${SharedEcsClusterName}/services)\n- [Shared VPC](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#VpcDetails:VpcId=${SharedVpcId}) | [Subnets](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#subnets:search=${SharedVpcId};sort=tag:Name) | [NACLs](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#acls:search=${SharedVpcId};sort=tag:Name) | [Peering](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#PeeringConnections:search=${SharedVpcId})\n- [ENIs](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#NIC:vpcId=${SharedVpcId})\n- [Security groups](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#securityGroups:vpc-id=${SharedVpcId};sort=tag:Name)\n- [Shared VPC Flow Logs Insights](https://console.aws.amazon.com/cloudwatch/home?region=${AWS::Region}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-1800$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20action*2c*20interfaceId*2c*20srcAddr*2c*20srcPort*2c*20dstAddr*2c*20dstPort*0a*7c*20sort*20*40timestamp*20desc*0a*7c*20limit*20500$257EisLiveTail$257Efalse$257EqueryId$257E$252738782a19-012d-4a8b-bda2-202bec5ce7e1$257Esource$257E$2528$257E$2527${SharedVpcFlowLogsLogGroupName}$2529$2529)\n- [EC2 Instances](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#Instances:instanceState=running;search=${SharedVpcId};sort=launchTime) | [Shared ASG](https://console.aws.amazon.com/ec2autoscaling/home?region=${AWS::Region}#/details/${SharedEcsAsgName}?view=details) | [Shared ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${SharedAlbName};sort=loadBalancerName)\n- [Dovetail ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${DovetailAlbName};sort=loadBalancerName) | [Exchange FTP NLB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${ExchangeFtpServerNlbName};sort=loadBalancerName) \n- [Shared Memcached](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#memcached-detail:id=${SharedMemcachedCacheName}) | [Shared Redis](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#redis-shards:redis-id=${SharedRedisReplicationGroupName})\n- [CMS Elasticsearch Service](https://console.aws.amazon.com/esv3/home?region=us-east-1#opensearch/domains/${CmsElasticsearchDomainName})\n- [Castle PostgreSQL](https://console.aws.amazon.com/rds/home?region=${AWS::Region}#database:id=${CastlePostgresInstanceId};is-cluster=false)\n" + "markdown": "\n# AWS Console\n\n- [CloudFormation root stack](https://console.aws.amazon.com/cloudformation/home?region=${AWS::Region}#/stacks/stackinfo?stackId=${RootStackId})\n- [Stack resource group](https://console.aws.amazon.com/resource-groups/group/${StackResourceGroupName}?region=${AWS::Region}) | [Logs](https://console.aws.amazon.com/resource-groups/group/${StackLogGroupsGroupName}?region=${AWS::Region})\n- [ECS Cluster](https://${AWS::Region}.console.aws.amazon.com/ecs/v2/clusters/${SharedEcsClusterName}/services?region=${AWS::Region}\n- [Shared VPC](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#VpcDetails:VpcId=${SharedVpcId}) | [Subnets](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#subnets:search=${SharedVpcId};sort=tag:Name) | [NACLs](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#acls:search=${SharedVpcId};sort=tag:Name) | [Peering](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#PeeringConnections:search=${SharedVpcId})\n- [ENIs](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#NIC:vpcId=${SharedVpcId})\n- [Security groups](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#securityGroups:vpc-id=${SharedVpcId};sort=tag:Name)\n- [Shared VPC Flow Logs Insights](https://console.aws.amazon.com/cloudwatch/home?region=${AWS::Region}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-1800$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20action*2c*20interfaceId*2c*20srcAddr*2c*20srcPort*2c*20dstAddr*2c*20dstPort*0a*7c*20sort*20*40timestamp*20desc*0a*7c*20limit*20500$257EisLiveTail$257Efalse$257EqueryId$257E$252738782a19-012d-4a8b-bda2-202bec5ce7e1$257Esource$257E$2528$257E$2527${SharedVpcFlowLogsLogGroupName}$2529$2529)\n- [EC2 Instances](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#Instances:instanceState=running;search=${SharedVpcId};sort=launchTime) | [Shared ASG](https://console.aws.amazon.com/ec2autoscaling/home?region=${AWS::Region}#/details/${SharedEcsAsgName}?view=details) | [Shared ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${SharedAlbName};sort=loadBalancerName)\n- [Dovetail ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${DovetailAlbName};sort=loadBalancerName) | [Exchange FTP NLB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${ExchangeFtpServerNlbName};sort=loadBalancerName) \n- [Shared Memcached](https://${AWS::Region}.console.aws.amazon.com/elasticache/home?region=${AWS::Region}#/memcached/${SharedMemcachedCacheName}) | [App Cache Redis](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#redis-shards:redis-id=${SharedAppRedisCacheName})\n- [Dovetail Redis](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#redis-shards:redis-id=${DovetailRedisReplicationGroupName})\n- [CMS Elasticsearch Service](https://${AWS::Region}.console.aws.amazon.com/aos/home?region=${AWS::Region}#opensearch/domains/${CmsElasticsearchDomainName})\n- [Castle PostgreSQL](https://console.aws.amazon.com/rds/home?region=${AWS::Region}#database:id=${CastlePostgresInstanceId};is-cluster=false)\n" } }, { diff --git a/spire/templates/root.yml b/spire/templates/root.yml index 89c792f6b..72d594176 100644 --- a/spire/templates/root.yml +++ b/spire/templates/root.yml @@ -513,6 +513,7 @@ Resources: TemplateURL: !Sub ${TemplateUrlBase}/spire/templates/shared-redis/security-groups.yml TimeoutInMinutes: 20 # Requires VPC and Redis SG + # This would more accurately be named "SharedDovetailRedisStack" SharedRedisStack: Type: AWS::CloudFormation::Stack DeletionPolicy: Delete @@ -902,7 +903,8 @@ Resources: SharedAlbFullName: !GetAtt SharedAlbStack.Outputs.AlbFullName SharedMemcachedCacheName: !GetAtt SharedMemcachedStack.Outputs.CacheName - SharedRedisReplicationGroupName: !GetAtt SharedRedisStack.Outputs.ReplicationGroupName + SharedAppRedisCacheName: !GetAtt SharedAppRedisStack.Outputs.CacheName + DovetailRedisReplicationGroupName: !GetAtt SharedRedisStack.Outputs.ReplicationGroupName # TODO StackResourceGroupName: !If From e7c19c071e7f19cd7400e7f5b24f5ac5276d5c6c Mon Sep 17 00:00:00 2001 From: Chris Kalafarski Date: Tue, 19 Sep 2023 15:17:40 -0400 Subject: [PATCH 8/9] Remove unnecessary quotes --- spire/templates/root.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spire/templates/root.yml b/spire/templates/root.yml index 72d594176..42ebeeb0e 100644 --- a/spire/templates/root.yml +++ b/spire/templates/root.yml @@ -472,7 +472,7 @@ Resources: Properties: Parameters: EnvironmentType: !Ref EnvironmentType - RegionMode: !FindInMap [RegionModeMap, !Ref "AWS::Region", !Ref EnvironmentType] + RegionMode: !FindInMap [RegionModeMap, !Ref AWS::Region, !Ref EnvironmentType] RootStackId: !Ref AWS::StackId RootStackName: !Ref AWS::StackName SourceSecurityGroupId1: !GetAtt SharedEcsAsgSecurityGroupStack.Outputs.InstanceSecurityGroupId From aac3ce729e82bfea974a26b219f91c7fa9908ed7 Mon Sep 17 00:00:00 2001 From: cavis Date: Tue, 19 Sep 2023 13:43:21 -0600 Subject: [PATCH 9/9] Start with redis 7 --- spire/templates/shared-app-redis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spire/templates/shared-app-redis.yml b/spire/templates/shared-app-redis.yml index 9d977114a..169fe2fed 100644 --- a/spire/templates/shared-app-redis.yml +++ b/spire/templates/shared-app-redis.yml @@ -82,7 +82,7 @@ Resources: - !Ref StagingInstanceType CacheSubnetGroupName: !Ref SharedAppRedisSubnetGroup Engine: redis - EngineVersion: 6.x + EngineVersion: 7.x NumCacheNodes: 1 PreferredMaintenanceWindow: sun:09:00-sun:10:00 Tags: