Added lambda to index events from s3 data lake to metrics cluster

Signed-off-by: Brandon Shien <[email protected]>

build.gradle

@@ -34,6 +34,8 @@ dependencies {
 
     implementation 'com.amazonaws:aws-java-sdk-secretsmanager:1.12.671'
 
+    implementation 'software.amazon.awssdk:s3:2.28.17'
+
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
 

infrastructure/lib/infrastructure-stack.ts

@@ -75,12 +75,17 @@ export class InfrastructureStack extends Stack {
           new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
         ]
       },
-      githubAutomationAppAccess: gitHubAutomationApp.githubAppRole.roleArn
+      githubAutomationAppAccess: gitHubAutomationApp.githubAppRole.roleArn,
+      githubEventsBucket: openSearchEventsS3Bucket.bucket,
     });
 
     // Create OpenSearch Metrics Lambda setup
     const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', {
-      opensearchDomainStack: openSearchDomainStack, vpcStack: vpcStack, lambdaPackage: Project.LAMBDA_PACKAGE
+      region: Project.REGION,
+      opensearchDomainStack: openSearchDomainStack,
+      vpcStack: vpcStack,
+      lambdaPackage: Project.LAMBDA_PACKAGE,
+      githubEventsBucket: openSearchEventsS3Bucket.bucket
     })
     openSearchMetricsWorkflowStack.node.addDependency(vpcStack, openSearchDomainStack);
 

infrastructure/lib/stacks/metricsWorkflow.ts

@@ -15,15 +15,20 @@ import { Construct } from 'constructs';
 import { OpenSearchLambda } from "../constructs/lambda";
 import { OpenSearchDomainStack } from "./opensearch";
 import { VpcStack } from "./vpc";
+import {Bucket} from "aws-cdk-lib/aws-s3";
 
 export interface OpenSearchMetricsStackProps extends StackProps {
+    readonly region: string;
     readonly opensearchDomainStack: OpenSearchDomainStack;
     readonly vpcStack: VpcStack;
-    readonly lambdaPackage: string
+    readonly lambdaPackage: string;
+    readonly githubEventsBucket: Bucket;
 }
 
+
 export interface WorkflowComponent {
-    opensearchMetricsWorkflowStateMachineName: string
+    opensearchMetricsWorkflowStateMachineName: string,
+    opensearchS3EventIndexWorkflowStateMachineName: string
 }
 export class OpenSearchMetricsWorkflowStack extends Stack {
     public readonly workflowComponent: WorkflowComponent;
@@ -36,6 +41,14 @@ export class OpenSearchMetricsWorkflowStack extends Stack {
             props.vpcStack,
             props.lambdaPackage
         );
+
+        const s3EventIndexTask = this.createS3EventIndexTask(this,
+            props.region,
+            props.opensearchDomainStack,
+            props.vpcStack,
+            props.lambdaPackage,
+            props.githubEventsBucket);
+
         const opensearchMetricsWorkflow = new StateMachine(this, 'OpenSearchMetricsWorkflow', {
             definition: metricsTask,
             timeout: Duration.minutes(15),
@@ -47,8 +60,20 @@ export class OpenSearchMetricsWorkflowStack extends Stack {
             targets: [new SfnStateMachine(opensearchMetricsWorkflow)],
         });
 
+        const opensearchS3EventIndexWorkflow = new StateMachine(this, 'OpenSearchS3EventIndexWorkflow', {
+            definition: s3EventIndexTask,
+            timeout: Duration.minutes(15),
+            stateMachineName: 'OpenSearchS3EventIndexWorkflow'
+        })
+
+        new Rule(this, 'OpenSearchS3EventIndexWorkflow-Every-Day', {
+            schedule: Schedule.expression('cron(0 0 * * ? *)'),
+            targets: [new SfnStateMachine(opensearchS3EventIndexWorkflow)],
+        });
+
         this.workflowComponent = {
-            opensearchMetricsWorkflowStateMachineName: opensearchMetricsWorkflow.stateMachineName
+            opensearchMetricsWorkflowStateMachineName: opensearchMetricsWorkflow.stateMachineName,
+            opensearchS3EventIndexWorkflowStateMachineName: opensearchS3EventIndexWorkflow.stateMachineName
         }
     }
 
@@ -61,7 +86,7 @@ export class OpenSearchMetricsWorkflowStack extends Stack {
             lambdaZipPath: `../../../build/distributions/${lambdaPackage}`,
             vpc: vpcStack.vpc,
             securityGroup: vpcStack.securityGroup,
-            role: opensearchDomainStack.openSearchLambdaRole,
+            role: opensearchDomainStack.openSearchMetricsLambdaRole,
             environment: {
                 OPENSEARCH_DOMAIN_ENDPOINT: openSearchDomain.domainEndpoint,
                 OPENSEARCH_DOMAIN_REGION: openSearchDomain.env.region,
@@ -74,4 +99,28 @@ export class OpenSearchMetricsWorkflowStack extends Stack {
             timeout: Duration.minutes(15)
         }).addRetry();
     }
+
+    private createS3EventIndexTask(scope: Construct, region: string, opensearchDomainStack: OpenSearchDomainStack, vpcStack: VpcStack, lambdaPackage: string, githubEventsBucket: Bucket) {
+        const openSearchDomain = opensearchDomainStack.domain;
+        const s3EventIndexLambda = new OpenSearchLambda(this, "OpenSearchMetricsS3EventIndexLambdaFunction", {
+            lambdaNameBase: "OpenSearchMetricsS3EventIndex",
+            handler: "org.opensearchmetrics.lambda.GithubEventsLambda",
+            lambdaZipPath: `../../../build/distributions/${lambdaPackage}`,
+            vpc: vpcStack.vpc,
+            securityGroup: vpcStack.securityGroup,
+            role: opensearchDomainStack.openSearchS3EventsIndexLambdaRole,
+            environment: {
+                S3_BUCKET_REGION: region,
+                EVENT_BUCKET_NAME: githubEventsBucket.bucketName,
+                OPENSEARCH_DOMAIN_ENDPOINT: openSearchDomain.domainEndpoint,
+                OPENSEARCH_DOMAIN_REGION: openSearchDomain.env.region,
+                OPENSEARCH_DOMAIN_ROLE: opensearchDomainStack.fullAccessRole.roleArn,
+            }
+        }).lambda;
+        return new LambdaInvoke(scope, 'S3 Event Index Lambda', {
+            lambdaFunction: s3EventIndexLambda,
+            resultPath: JsonPath.DISCARD,
+            timeout: Duration.minutes(15)
+        }).addRetry();
+    }
 }

infrastructure/lib/stacks/monitoringDashboard.ts

@@ -70,6 +70,7 @@ export class OpenSearchMetricsMonitoringStack extends Stack {
     private snsMonitorStepFunctionExecutionsFailed(): void {
         const stepFunctionSnsAlarms = [
             { alertName: 'StepFunction_execution_errors_MetricsWorkflow', stateMachineName: this.props.workflowComponent.opensearchMetricsWorkflowStateMachineName },
+            { alertName: 'StepFunction_execution_errors_S3EventIndexWorkflow', stateMachineName: this.props.workflowComponent.opensearchS3EventIndexWorkflowStateMachineName },
         ];
 
         new StepFunctionSns(this, "SnsMonitors-StepFunctionExecutionsFailed", {

infrastructure/lib/stacks/opensearch.ts

@@ -15,6 +15,7 @@ import { OpenSearchMetricsNginxCognito } from "../constructs/opensearchNginxProx
 import Project from "../enums/project";
 import { OpenSearchHealthRoute53 } from "./route53";
 import { VpcStack } from "./vpc";
+import {Bucket} from "aws-cdk-lib/aws-s3";
 
 
 export interface OpenSearchStackProps {
@@ -24,6 +25,7 @@ export interface OpenSearchStackProps {
     readonly enableNginxCognito: boolean;
     readonly jenkinsAccess?: jenkinsAccess;
     readonly githubAutomationAppAccess?: string;
+    readonly githubEventsBucket: Bucket;
 }
 
 
@@ -43,18 +45,19 @@ export class OpenSearchDomainStack extends Stack {
     public readonly domain: Domain;
     public readonly props: OpenSearchStackProps;
     public readonly fullAccessRole: IRole;
-    public readonly openSearchLambdaRole: IRole;
+    public readonly openSearchMetricsLambdaRole: IRole;
+    public readonly openSearchS3EventsIndexLambdaRole: IRole;
     public readonly opensearchDomainConfig: OpenSearchDomainConfig;
 
     constructor(scope: Construct, id: string, props: OpenSearchStackProps) {
         super(scope, id);
         this.props = props;
 
 
-        this.openSearchLambdaRole = new Role(this, 'OpenSearchDomainLambdaRole', {
+        this.openSearchMetricsLambdaRole = new Role(this, 'OpenSearchDomainLambdaRole', {
             assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
             description: "OpenSearch Metrics Lambda Execution Role",
-            roleName: "OpenSearchLambdaRole",
+            roleName: "OpenSearchMetricsLambdaRole",
 
             inlinePolicies: {
                 "opensearchAssumeRolePolicy": new PolicyDocument({
@@ -76,6 +79,41 @@ export class OpenSearchDomainStack extends Stack {
             ]
         });
 
+        this.openSearchS3EventsIndexLambdaRole = new Role(this, 'OpenSearchS3EventIndexLambdaRole', {
+            assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
+            description: "OpenSearch Metrics S3 Event Index Lambda Execution Role",
+            roleName: "OpenSearchS3EventIndexLambdaRole",
+            inlinePolicies: {
+                "opensearchAssumeRolePolicy": new PolicyDocument({
+                    statements: [
+                        new PolicyStatement({
+                            effect: Effect.ALLOW,
+                            actions: ["sts:AssumeRole"],
+                            resources: [`arn:aws:iam::${props.account}:role/OpenSearchFullAccessRole`],
+                            conditions: {
+                                StringEquals: { 'aws:PrincipalAccount': props.account, 'aws:RequestedRegion': props.region, },
+                            }
+                        })
+                    ]
+                }),
+                "opensearchReadS3EventsPolicy": new PolicyDocument({
+                    statements: [
+                        new PolicyStatement({
+                            effect: Effect.ALLOW,
+                            actions: ["s3:GetObject",
+                                "s3:ListBucket"],
+                            resources: [props.githubEventsBucket.bucketArn,
+                                `${props.githubEventsBucket.bucketArn}/*`],
+                        })
+                    ]
+                })
+            },
+            managedPolicies: [
+                ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole'),
+                ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole'),
+            ]
+        });
+
         this.opensearchDomainConfig = {
             domainName: 'opensearch-health',
             ebsOptions: {
@@ -85,7 +123,7 @@ export class OpenSearchDomainStack extends Stack {
 
         const domainArn = `arn:aws:es:${props.region}:${props.account}:domain/${this.opensearchDomainConfig.domainName}/*`;
 
-        const secureRolesList = [this.openSearchLambdaRole]
+        const secureRolesList = [this.openSearchMetricsLambdaRole, this.openSearchS3EventsIndexLambdaRole]
         this.fullAccessRole = new Role(this, 'OpenSearchFullAccessRole', {
             assumedBy: new CompositePrincipal(...secureRolesList.map((role) => new ArnPrincipal(role.roleArn))),
             description: "Master role for OpenSearch full access",

infrastructure/test/monitoring-stack.test.ts

@@ -15,25 +15,31 @@ import { VpcStack } from "../lib/stacks/vpc";
 import { ArnPrincipal } from "aws-cdk-lib/aws-iam";
 import { OpenSearchMetricsMonitoringStack } from "../lib/stacks/monitoringDashboard";
 import { OpenSearchMetricsSecretsStack } from "../lib/stacks/secrets";
+import {OpenSearchS3} from "../lib/stacks/s3";
 
 test('Monitoring Stack Test', () => {
     const app = new App();
     const vpcStack = new VpcStack(app, 'OpenSearchHealth-VPC', {});
+        const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3");
+    const opensearchDomainStack = new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', {
+        region: "us-east-1",
+        account: "test-account",
+        vpcStack: vpcStack,
+        enableNginxCognito: true,
+        jenkinsAccess: {
+            jenkinsAccountRoles: [
+                new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
+                new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
+            ]
+        },
+        githubEventsBucket: s3Stack.bucket
+    });
     const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', {
-        opensearchDomainStack: new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', {
-            region: "us-east-1",
-            account: "test-account",
-            vpcStack: vpcStack,
-            enableNginxCognito: true,
-            jenkinsAccess: {
-                jenkinsAccountRoles: [
-                    new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
-                    new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
-                ]
-            }
-        }),
+        region: Project.REGION,
+        opensearchDomainStack: opensearchDomainStack,
         vpcStack: vpcStack,
-        lambdaPackage: Project.LAMBDA_PACKAGE
+        lambdaPackage: Project.LAMBDA_PACKAGE,
+        githubEventsBucket: s3Stack.bucket
     });
     const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", {
         secretName: 'metrics-creds'
@@ -49,7 +55,7 @@ test('Monitoring Stack Test', () => {
     const template = Template.fromStack(openSearchMetricsMonitoringStack);
     template.resourceCountIs('AWS::IAM::Role', 2);
     template.resourceCountIs('AWS::IAM::Policy', 1);
-    template.resourceCountIs('AWS::CloudWatch::Alarm', 2);
+    template.resourceCountIs('AWS::CloudWatch::Alarm', 3);
     template.resourceCountIs('AWS::SNS::Topic', 2);
     template.resourceCountIs('AWS::Synthetics::Canary', 1);
     template.hasResourceProperties('AWS::IAM::Role', {
@@ -156,6 +162,40 @@ test('Monitoring Stack Test', () => {
         "Threshold": 1,
         "TreatMissingData": "notBreaching"
     });
+    template.hasResourceProperties('AWS::CloudWatch::Alarm', {
+        "AlarmActions": [
+            {
+                "Ref": "SnsMonitorsStepFunctionExecutionsFailedOpenSearchMetricsAlarmStepFunctionExecutionsFailed0B259DBC"
+            }
+        ],
+        "AlarmDescription": "Detect SF execution failure",
+        "AlarmName": "StepFunction_execution_errors_S3EventIndexWorkflow",
+        "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+        "DatapointsToAlarm": 1,
+        "Dimensions": [
+            {
+                "Name": "StateMachineArn",
+                "Value": {
+                    "Fn::Join": [
+                        "",
+                        [
+                            "arn:aws:states:::stateMachine:",
+                            {
+                                "Fn::ImportValue": "OpenSearchMetrics-Workflow:ExportsOutputFnGetAttOpenSearchS3EventIndexWorkflow0C74BA9FName074F0965"
+                            }
+                        ]
+                    ]
+                }
+            }
+        ],
+        "EvaluationPeriods": 1,
+        "MetricName": "ExecutionsFailed",
+        "Namespace": "AWS/States",
+        "Period": 300,
+        "Statistic": "Sum",
+        "Threshold": 1,
+        "TreatMissingData": "notBreaching"
+    });
     template.hasResourceProperties('AWS::Synthetics::Canary', {
         "ArtifactS3Location": {
             "Fn::Join": [

infrastructure/test/nginx.test.ts

@@ -14,10 +14,12 @@ import Project from "../lib/enums/project";
 import { OpenSearchDomainStack } from "../lib/stacks/opensearch";
 import { ArnPrincipal } from "aws-cdk-lib/aws-iam";
 import { OpenSearchHealthRoute53 } from "../lib/stacks/route53";
+import {OpenSearchS3} from "../lib/stacks/s3";
 
 test('OpenSearchMetricsNginxReadonly Stack Test', () => {
     const app = new App();
     const vpcStack = new VpcStack(app, "OpenSearchHealth-VPC", {});
+    const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3");
     const openSearchDomainStack = new OpenSearchDomainStack(app, "OpenSearchHealth-OpenSearch", {
         region: Project.REGION,
         account: Project.AWS_ACCOUNT,
@@ -28,7 +30,8 @@ test('OpenSearchMetricsNginxReadonly Stack Test', () => {
                 new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
                 new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
             ]
-        }
+        },
+        githubEventsBucket: s3Stack.bucket
     });
     const metricsHostedZone = new OpenSearchHealthRoute53(app, "OpenSearchMetrics-HostedZone", {
         hostedZone: Project.METRICS_HOSTED_ZONE,