Merge pull request #705 from PRX/feat/shared_rails_app_redis

Setup shared app redis
PRX · Sep 19, 2023 · 95d90c8 · 95d90c8
2 parents 0126f0f + aac3ce7
commit 95d90c8
Show file tree

Hide file tree

Showing 7 changed files with 170 additions and 87 deletions.
diff --git a/spire/templates/apps-200A.yml b/spire/templates/apps-200A.yml
@@ -29,6 +29,8 @@ Parameters:
   VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id }
   VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id }
   VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id }
+  SharedAppRedisEndpointAddress: { Type: String }
+  SharedAppRedisEndpointPort: { Type: String }
   SharedMemcachedEndpointAddress: { Type: String }
   SharedMemcachedEndpointPort: { Type: String }
   SharedRedisReplicationGroupEndpointAddress: { Type: String }
@@ -274,6 +276,8 @@ Resources:
         S3SigningUserName: !Ref S3SigningUserName
         S3SigningEndpointUrl: !Ref S3SigningEndpointUrl
         S3SigningAccessKeyId: !Ref S3SigningAccessKeyId
+        SharedAppRedisEndpointAddress: !Ref SharedAppRedisEndpointAddress
+        SharedAppRedisEndpointPort: !Ref SharedAppRedisEndpointPort
         SharedAuroraPostgresqlEndpoint: !Ref SharedAuroraPostgresqlEndpoint
         SharedAuroraPostgresqlPort: !Ref SharedAuroraPostgresqlPort
         PorterJobExecutionSnsTopicArn: !Ref PorterJobExecutionSnsTopicArn

diff --git a/spire/templates/apps-300A.yml b/spire/templates/apps-300A.yml
@@ -20,9 +20,6 @@ Parameters:
   RootStackId: { Type: String }
   VpcId: { Type: AWS::EC2::VPC::Id }
   NewRelicApiKeyPrxLite: { Type: String }
-  VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id }
-  VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id }
-  VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id }
   VpcPublicSubnet1Id: { Type: AWS::EC2::Subnet::Id }
   VpcPublicSubnet2Id: { Type: AWS::EC2::Subnet::Id }
   VpcPublicSubnet3Id: { Type: AWS::EC2::Subnet::Id }
@@ -34,6 +31,8 @@ Parameters:
   SharedEcsAsgInstanceSecurityGroupId: { Type: AWS::EC2::SecurityGroup::Id }
   SharedRedisReplicationGroupEndpointAddress: { Type: String }
   SharedRedisReplicationGroupEndpointPort: { Type: String }
+  SharedAppRedisEndpointAddress: { Type: String }
+  SharedAppRedisEndpointPort: { Type: String }
   SharedAuroraPostgresqlEndpoint: { Type: String }
   SharedAuroraPostgresqlPort: { Type: String }
   SharedPostgresqlClientSecurityGroupId: { Type: String }
@@ -83,17 +82,15 @@ Resources:
         RootStackId: !Ref RootStackId
         EchoServiceToken: !Ref EchoServiceToken
         CloudWatchAlarmTaggerServiceToken: !Ref CloudWatchAlarmTaggerServiceToken
-        VpcPrivateSubnet1Id: !Ref VpcPrivateSubnet1Id
-        VpcPrivateSubnet2Id: !Ref VpcPrivateSubnet2Id
-        VpcPrivateSubnet3Id: !Ref VpcPrivateSubnet3Id
         VpcPublicSubnet1Id: !Ref VpcPublicSubnet1Id
         VpcPublicSubnet2Id: !Ref VpcPublicSubnet2Id
         VpcPublicSubnet3Id: !Ref VpcPublicSubnet3Id
         EcsLaunchEndpointsAccessSecurityGroupId: !Ref EcsLaunchEndpointsAccessSecurityGroupId
         KmsEndpointAccessSecurityGroupId: !Ref KmsEndpointAccessSecurityGroupId
+        SharedAppRedisEndpointAddress: !Ref SharedAppRedisEndpointAddress
+        SharedAppRedisEndpointPort: !Ref SharedAppRedisEndpointPort
         SharedAuroraPostgresqlEndpoint: !Ref SharedAuroraPostgresqlEndpoint
         SharedAuroraPostgresqlPort: !Ref SharedAuroraPostgresqlPort
-        SharedEcsAsgInstanceSecurityGroupId: !Ref SharedEcsAsgInstanceSecurityGroupId
         SharedPostgresqlClientSecurityGroupId: !Ref SharedPostgresqlClientSecurityGroupId
         CastlePostgresInstanceEndpointAddress: !Ref CastlePostgresInstanceEndpointAddress
         CastlePostgresInstanceEndpointPort: !Ref CastlePostgresInstanceEndpointPort

diff --git a/spire/templates/apps/augury.yml b/spire/templates/apps/augury.yml
@@ -40,17 +40,15 @@ Parameters:
   NewRelicApiKeyPrxLite: { Type: String }
   EcrImageTag: { Type: AWS::SSM::Parameter::Value<String> }
   AlbListenerRulePriorityPrefix: { Type: String }
-  VpcPrivateSubnet1Id: { Type: AWS::EC2::Subnet::Id }
-  VpcPrivateSubnet2Id: { Type: AWS::EC2::Subnet::Id }
-  VpcPrivateSubnet3Id: { Type: AWS::EC2::Subnet::Id }
   VpcPublicSubnet1Id: { Type: AWS::EC2::Subnet::Id }
   VpcPublicSubnet2Id: { Type: AWS::EC2::Subnet::Id }
   VpcPublicSubnet3Id: { Type: AWS::EC2::Subnet::Id }
   EcsLaunchEndpointsAccessSecurityGroupId: { Type: AWS::EC2::SecurityGroup::Id }
   KmsEndpointAccessSecurityGroupId: { Type: AWS::EC2::SecurityGroup::Id }
+  SharedAppRedisEndpointAddress: { Type: String }
+  SharedAppRedisEndpointPort: { Type: String }
   SharedAuroraPostgresqlEndpoint: { Type: String }
   SharedAuroraPostgresqlPort: { Type: String }
-  SharedEcsAsgInstanceSecurityGroupId: { Type: String }
   SharedPostgresqlClientSecurityGroupId: { Type: String }
   CastlePostgresInstanceEndpointAddress: { Type: String }
   CastlePostgresInstanceEndpointPort: { Type: String }
@@ -84,70 +82,6 @@ Resources:
       SlowWorkerLoggedErrorsMetricName: !Sub SlowWorkerLoggedErrors${EnvironmentType}
       WebLoggedErrorsMetricName: !Sub WebLoggedErrors${EnvironmentType}
 
-  RedisSecurityGroup:
-    Type: AWS::EC2::SecurityGroup
-    Condition: IsPrimaryRegion
-    Properties:
-      GroupDescription: !Sub Augury ${EnvironmentType} Redis security group
-      SecurityGroupIngress:
-        - FromPort: 6379
-          IpProtocol: tcp
-          SourceSecurityGroupId: !Ref SharedEcsAsgInstanceSecurityGroupId
-          ToPort: 6379
-      Tags:
-        - { Key: Name, Value: !Sub "${RootStackName}_augury_redis" }
-        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
-        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
-        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
-        - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
-        - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
-        - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
-        - { Key: prx:dev:family, Value: Dovetail }
-        - { Key: prx:dev:application, Value: Augury }
-      VpcId: !Ref VpcId
-  RedisSubnetGroup:
-    Type: AWS::ElastiCache::SubnetGroup
-    Condition: IsPrimaryRegion
-    Properties:
-      Description: !Sub Augury ${EnvironmentType} Redis subnet group
-      SubnetIds:
-        - !Ref VpcPrivateSubnet1Id
-        - !Ref VpcPrivateSubnet2Id
-        - !Ref VpcPrivateSubnet3Id
-      Tags:
-        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
-        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
-        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
-        - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
-        - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
-        - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
-        - { Key: prx:dev:family, Value: Dovetail }
-        - { Key: prx:dev:application, Value: Augury }
-  RedisCluster:
-    Type: AWS::ElastiCache::CacheCluster
-    DeletionPolicy: Delete
-    UpdateReplacePolicy: Delete
-    Condition: IsPrimaryRegion
-    Properties:
-      AutoMinorVersionUpgrade: false
-      CacheNodeType: !If [IsProduction, cache.t4g.small, cache.t4g.micro]
-      CacheSubnetGroupName: !Ref RedisSubnetGroup
-      Engine: redis
-      EngineVersion: 6.x
-      NumCacheNodes: 1
-      PreferredMaintenanceWindow: sun:09:00-sun:10:00
-      Tags:
-        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
-        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
-        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
-        - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
-        - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
-        - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
-        - { Key: prx:dev:family, Value: Dovetail }
-        - { Key: prx:dev:application, Value: Augury }
-      VpcSecurityGroupIds:
-        - !Ref RedisSecurityGroup
-
   HostHeaderListenerRule:
     Type: AWS::ElasticLoadBalancingV2::ListenerRule
     Properties:
@@ -365,9 +299,9 @@ Resources:
             - Name: POSTGRES_PORT
               Value: !Ref SharedAuroraPostgresqlPort
             - Name: REDIS_HOST
-              Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"]
+              Value: !Ref SharedAppRedisEndpointAddress
             - Name: REDIS_PORT
-              Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"]
+              Value: !Ref SharedAppRedisEndpointPort
             - Name: UPLOAD_ADFILE_SECRET_ACCESS_KEY
               Value: !GetAtt S3SigningAccessKey.SecretAccessKey
             - Name: UPLOAD_ADFILE_ACCESS_KEY_ID
@@ -504,9 +438,9 @@ Resources:
             - Name: POSTGRES_PORT
               Value: !Ref SharedAuroraPostgresqlPort
             - Name: REDIS_HOST
-              Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"]
+              Value: !Ref SharedAppRedisEndpointAddress
             - Name: REDIS_PORT
-              Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"]
+              Value: !Ref SharedAppRedisEndpointPort
             - Name: RAILS_SERVE_STATIC_FILES
               Value: "true"
             - Name: POSTGRES_POOL_SIZE
@@ -998,9 +932,9 @@ Resources:
             - Name: POSTGRES_PORT
               Value: !Ref SharedAuroraPostgresqlPort
             - Name: REDIS_HOST
-              Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Address, !Ref "AWS::NoValue"]
+              Value: !Ref SharedAppRedisEndpointAddress
             - Name: REDIS_PORT
-              Value: !If [IsPrimaryRegion, !GetAtt RedisCluster.RedisEndpoint.Port, !Ref "AWS::NoValue"]
+              Value: !Ref SharedAppRedisEndpointPort
             - Name: RAILS_SERVE_STATIC_FILES
               Value: "true"
             - Name: POSTGRES_POOL_SIZE

diff --git a/spire/templates/apps/feeder.yml b/spire/templates/apps/feeder.yml
@@ -45,6 +45,8 @@ Parameters:
   NewRelicApiKeyPrxLite: { Type: String }
   EcrImageTag: { Type: AWS::SSM::Parameter::Value<String> }
   AlbListenerRulePriorityPrefix: { Type: String }
+  SharedAppRedisEndpointAddress: { Type: String }
+  SharedAppRedisEndpointPort: { Type: String }
   SharedAuroraPostgresqlEndpoint: { Type: String }
   SharedAuroraPostgresqlPort: { Type: String }
   AnnounceResourcePrefix: { Type: String }
@@ -673,6 +675,10 @@ Resources:
               Value: !GetAtt Constants.PublicFeedsUrlPrefix
             - Name: RAILS_ENV
               Value: !Ref EnvironmentTypeLowercase
+            - Name: REDIS_HOST
+              Value: !Ref SharedAppRedisEndpointAddress
+            - Name: REDIS_PORT
+              Value: !Ref SharedAppRedisEndpointPort
             - Name: UPLOAD_BUCKET_NAME
               Value: !Ref FeederUploadsBucket
             - Name: UPLOAD_S3_ENDPOINT_HOST
@@ -813,6 +819,10 @@ Resources:
               Value: !GetAtt Constants.PublicFeedsUrlPrefix
             - Name: RAILS_ENV
               Value: !Ref EnvironmentTypeLowercase
+            - Name: REDIS_HOST
+              Value: !Ref SharedAppRedisEndpointAddress
+            - Name: REDIS_PORT
+              Value: !Ref SharedAppRedisEndpointPort
             - Name: START_SAY_WHEN
               Value: true
             - Name: WORKER_COUNT

diff --git a/spire/templates/dashboards.yml b/spire/templates/dashboards.yml
@@ -25,7 +25,8 @@ Parameters:
   SharedVpcFlowLogsLogGroupName: { Type: String }
 
   SharedMemcachedCacheName: { Type: String }
-  SharedRedisReplicationGroupName: { Type: String }
+  SharedAppRedisCacheName: { Type: String }
+  DovetailRedisReplicationGroupName: { Type: String }
 
   StackResourceGroupName: { Type: String }
   StackLogGroupsGroupName: { Type: String }
@@ -97,7 +98,7 @@ Resources:
                     "x": 0,
                     "type": "text",
                     "properties": {
-                        "markdown": "\n# AWS Console\n\n- [CloudFormation root stack](https://console.aws.amazon.com/cloudformation/home?region=${AWS::Region}#/stacks/stackinfo?stackId=${RootStackId})\n- [Stack resource group](https://console.aws.amazon.com/resource-groups/group/${StackResourceGroupName}?region=${AWS::Region}) | [Logs](https://console.aws.amazon.com/resource-groups/group/${StackLogGroupsGroupName}?region=${AWS::Region})\n- [ECS Cluster](https://console.aws.amazon.com/ecs/home?region=${AWS::Region}#/clusters/${SharedEcsClusterName}/services)\n- [Shared VPC](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#VpcDetails:VpcId=${SharedVpcId}) | [Subnets](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#subnets:search=${SharedVpcId};sort=tag:Name) | [NACLs](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#acls:search=${SharedVpcId};sort=tag:Name) | [Peering](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#PeeringConnections:search=${SharedVpcId})\n- [ENIs](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#NIC:vpcId=${SharedVpcId})\n- [Security groups](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#securityGroups:vpc-id=${SharedVpcId};sort=tag:Name)\n- [Shared VPC Flow Logs Insights](https://console.aws.amazon.com/cloudwatch/home?region=${AWS::Region}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-1800$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20action*2c*20interfaceId*2c*20srcAddr*2c*20srcPort*2c*20dstAddr*2c*20dstPort*0a*7c*20sort*20*40timestamp*20desc*0a*7c*20limit*20500$257EisLiveTail$257Efalse$257EqueryId$257E$252738782a19-012d-4a8b-bda2-202bec5ce7e1$257Esource$257E$2528$257E$2527${SharedVpcFlowLogsLogGroupName}$2529$2529)\n- [EC2 Instances](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#Instances:instanceState=running;search=${SharedVpcId};sort=launchTime) | [Shared ASG](https://console.aws.amazon.com/ec2autoscaling/home?region=${AWS::Region}#/details/${SharedEcsAsgName}?view=details) | [Shared ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${SharedAlbName};sort=loadBalancerName)\n- [Dovetail ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${DovetailAlbName};sort=loadBalancerName) | [Exchange FTP NLB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${ExchangeFtpServerNlbName};sort=loadBalancerName) \n- [Shared Memcached](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#memcached-detail:id=${SharedMemcachedCacheName}) | [Shared Redis](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#redis-shards:redis-id=${SharedRedisReplicationGroupName})\n- [CMS Elasticsearch Service](https://console.aws.amazon.com/esv3/home?region=us-east-1#opensearch/domains/${CmsElasticsearchDomainName})\n- [Castle PostgreSQL](https://console.aws.amazon.com/rds/home?region=${AWS::Region}#database:id=${CastlePostgresInstanceId};is-cluster=false)\n"
+                        "markdown": "\n# AWS Console\n\n- [CloudFormation root stack](https://console.aws.amazon.com/cloudformation/home?region=${AWS::Region}#/stacks/stackinfo?stackId=${RootStackId})\n- [Stack resource group](https://console.aws.amazon.com/resource-groups/group/${StackResourceGroupName}?region=${AWS::Region}) | [Logs](https://console.aws.amazon.com/resource-groups/group/${StackLogGroupsGroupName}?region=${AWS::Region})\n- [ECS Cluster](https://${AWS::Region}.console.aws.amazon.com/ecs/v2/clusters/${SharedEcsClusterName}/services?region=${AWS::Region}\n- [Shared VPC](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#VpcDetails:VpcId=${SharedVpcId}) | [Subnets](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#subnets:search=${SharedVpcId};sort=tag:Name) | [NACLs](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#acls:search=${SharedVpcId};sort=tag:Name) | [Peering](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#PeeringConnections:search=${SharedVpcId})\n- [ENIs](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#NIC:vpcId=${SharedVpcId})\n- [Security groups](https://console.aws.amazon.com/vpc/home?region=${AWS::Region}#securityGroups:vpc-id=${SharedVpcId};sort=tag:Name)\n- [Shared VPC Flow Logs Insights](https://console.aws.amazon.com/cloudwatch/home?region=${AWS::Region}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-1800$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20action*2c*20interfaceId*2c*20srcAddr*2c*20srcPort*2c*20dstAddr*2c*20dstPort*0a*7c*20sort*20*40timestamp*20desc*0a*7c*20limit*20500$257EisLiveTail$257Efalse$257EqueryId$257E$252738782a19-012d-4a8b-bda2-202bec5ce7e1$257Esource$257E$2528$257E$2527${SharedVpcFlowLogsLogGroupName}$2529$2529)\n- [EC2 Instances](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#Instances:instanceState=running;search=${SharedVpcId};sort=launchTime) | [Shared ASG](https://console.aws.amazon.com/ec2autoscaling/home?region=${AWS::Region}#/details/${SharedEcsAsgName}?view=details) | [Shared ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${SharedAlbName};sort=loadBalancerName)\n- [Dovetail ALB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${DovetailAlbName};sort=loadBalancerName) | [Exchange FTP NLB](https://console.aws.amazon.com/ec2/v2/home?region=${AWS::Region}#LoadBalancers:search=${ExchangeFtpServerNlbName};sort=loadBalancerName) \n- [Shared Memcached](https://${AWS::Region}.console.aws.amazon.com/elasticache/home?region=${AWS::Region}#/memcached/${SharedMemcachedCacheName}) | [App Cache Redis](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#redis-shards:redis-id=${SharedAppRedisCacheName})\n- [Dovetail Redis](https://console.aws.amazon.com/elasticache/home?region=${AWS::Region}#redis-shards:redis-id=${DovetailRedisReplicationGroupName})\n- [CMS Elasticsearch Service](https://${AWS::Region}.console.aws.amazon.com/aos/home?region=${AWS::Region}#opensearch/domains/${CmsElasticsearchDomainName})\n- [Castle PostgreSQL](https://console.aws.amazon.com/rds/home?region=${AWS::Region}#database:id=${CastlePostgresInstanceId};is-cluster=false)\n"
                     }
                 },
                 {