Added workflow to index events from s3 data lake to metrics cluster

DEVELOPER_GUIDE.md

@@ -26,6 +26,7 @@ So you want to contribute code to this project? Excellent! We're glad you're her
     - `cdk deploy OpenSearchMetrics-GitHubWorkflowMonitor-Alarms`: Creates the Alarms to Monitor the Critical GitHub CI workflows by the GitHub Automation App.
     - `cdk deploy OpenSearchMetrics-GitHubAutomationApp`: Create the resources which launches the [GitHub Automation App](https://github.com/opensearch-project/automation-app). Listens to GitHub events and index the data to Metrics cluster.
     - `cdk deploy OpenSearchMetrics-GitHubAutomationAppEvents-S3`: Creates the S3 Bucket for the [GitHub Automation App](https://github.com/opensearch-project/automation-app) to store OpenSearch Project GitHub Events.
+    - `cdk deploy OpenSearchS3EventIndex-Workflow`: Creates the Lambda and Step Function to index the GitHub Events stored in the S3 Bucket to the Metrics cluster.
 
 ### Forking and Cloning
 

build.gradle

@@ -34,6 +34,8 @@ dependencies {
 
     implementation 'com.amazonaws:aws-java-sdk-secretsmanager:1.12.671'
 
+    implementation 'software.amazon.awssdk:s3:2.28.17'
+
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
 

infrastructure/lib/infrastructure-stack.ts

@@ -22,6 +22,7 @@ import { OpenSearchMetricsSecretsStack } from "./stacks/secrets";
 import { GitHubAutomationApp } from "./stacks/gitHubAutomationApp";
 import { OpenSearchS3 } from "./stacks/s3";
 import { GitHubWorkflowMonitorAlarms } from "./stacks/gitHubWorkflowMonitorAlarms";
+import {OpenSearchS3EventIndexWorkflowStack} from "./stacks/s3EventIndexWorkflow";
 
 export class InfrastructureStack extends Stack {
   constructor(scope: Construct, id: string, props?: StackProps) {
@@ -75,15 +76,28 @@ export class InfrastructureStack extends Stack {
           new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
         ]
       },
-      githubAutomationAppAccess: gitHubAutomationApp.githubAppRole.roleArn
+      githubAutomationAppAccess: gitHubAutomationApp.githubAppRole.roleArn,
+      githubEventsBucket: openSearchEventsS3Bucket.bucket,
     });
 
     // Create OpenSearch Metrics Lambda setup
     const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', {
-      opensearchDomainStack: openSearchDomainStack, vpcStack: vpcStack, lambdaPackage: Project.LAMBDA_PACKAGE
+      opensearchDomainStack: openSearchDomainStack,
+      vpcStack: vpcStack,
+      lambdaPackage: Project.LAMBDA_PACKAGE,
     })
     openSearchMetricsWorkflowStack.node.addDependency(vpcStack, openSearchDomainStack);
 
+    // Create OpenSearch S3 Event Index Lambda setup
+    const openSearchS3EventIndexWorkflowStack = new OpenSearchS3EventIndexWorkflowStack(app, 'OpenSearchS3EventIndex-Workflow', {
+      region: Project.REGION,
+      opensearchDomainStack: openSearchDomainStack,
+      vpcStack: vpcStack,
+      lambdaPackage: Project.LAMBDA_PACKAGE,
+      githubEventsBucket: openSearchEventsS3Bucket.bucket
+    })
+    openSearchS3EventIndexWorkflowStack.node.addDependency(vpcStack, openSearchDomainStack);
+
     // Create Secret Manager for the metrics project
     const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", {
       secretName: 'metrics-creds'
@@ -94,7 +108,10 @@ export class InfrastructureStack extends Stack {
     const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", {
       region: Project.REGION,
       account: Project.AWS_ACCOUNT,
-      workflowComponent: openSearchMetricsWorkflowStack.workflowComponent,
+      workflowComponent: {
+        opensearchMetricsWorkflowStateMachineName: openSearchMetricsWorkflowStack.workflowComponent.opensearchMetricsWorkflowStateMachineName,
+        opensearchS3EventIndexWorkflowStateMachineName: openSearchS3EventIndexWorkflowStack.workflowComponent.opensearchS3EventIndexWorkflowStateMachineName
+      },
       lambdaPackage: Project.LAMBDA_PACKAGE,
       secrets: openSearchMetricsSecretsStack.secret,
       vpcStack: vpcStack

infrastructure/lib/stacks/metricsWorkflow.ts

@@ -53,15 +53,15 @@ export class OpenSearchMetricsWorkflowStack extends Stack {
     }
 
     private createMetricsTask(scope: Construct, opensearchDomainStack: OpenSearchDomainStack,
-        vpcStack: VpcStack, lambdaPackage: string) {
+                              vpcStack: VpcStack, lambdaPackage: string) {
         const openSearchDomain = opensearchDomainStack.domain;
         const metricsLambda = new OpenSearchLambda(scope, "OpenSearchMetricsLambdaFunction", {
             lambdaNameBase: "OpenSearchMetricsDashboards",
             handler: "org.opensearchmetrics.lambda.MetricsLambda",
             lambdaZipPath: `../../../build/distributions/${lambdaPackage}`,
             vpc: vpcStack.vpc,
             securityGroup: vpcStack.securityGroup,
-            role: opensearchDomainStack.openSearchLambdaRole,
+            role: opensearchDomainStack.openSearchMetricsLambdaRole,
             environment: {
                 OPENSEARCH_DOMAIN_ENDPOINT: openSearchDomain.domainEndpoint,
                 OPENSEARCH_DOMAIN_REGION: openSearchDomain.env.region,

infrastructure/lib/stacks/monitoringDashboard.ts

@@ -16,14 +16,13 @@ import { canarySns } from "../constructs/canarySns";
 import { OpenSearchLambda } from "../constructs/lambda";
 import { StepFunctionSns } from "../constructs/stepFunctionSns";
 import Project from "../enums/project";
-import { WorkflowComponent } from "./metricsWorkflow";
 import { VpcStack } from "./vpc";
 
 
 interface OpenSearchMetricsMonitoringStackProps extends StackProps {
     readonly region: string;
     readonly account: string;
-    readonly workflowComponent: WorkflowComponent;
+    readonly workflowComponent: {[component: string]: string};
     readonly lambdaPackage: string;
     readonly secrets: Secret;
     readonly vpcStack: VpcStack;
@@ -70,6 +69,7 @@ export class OpenSearchMetricsMonitoringStack extends Stack {
     private snsMonitorStepFunctionExecutionsFailed(): void {
         const stepFunctionSnsAlarms = [
             { alertName: 'StepFunction_execution_errors_MetricsWorkflow', stateMachineName: this.props.workflowComponent.opensearchMetricsWorkflowStateMachineName },
+            { alertName: 'StepFunction_execution_errors_S3EventIndexWorkflow', stateMachineName: this.props.workflowComponent.opensearchS3EventIndexWorkflowStateMachineName },
         ];
 
         new StepFunctionSns(this, "SnsMonitors-StepFunctionExecutionsFailed", {

infrastructure/lib/stacks/opensearch.ts

@@ -15,6 +15,7 @@ import { OpenSearchMetricsNginxCognito } from "../constructs/opensearchNginxProx
 import Project from "../enums/project";
 import { OpenSearchHealthRoute53 } from "./route53";
 import { VpcStack } from "./vpc";
+import {Bucket} from "aws-cdk-lib/aws-s3";
 
 
 export interface OpenSearchStackProps {
@@ -24,6 +25,7 @@ export interface OpenSearchStackProps {
     readonly enableNginxCognito: boolean;
     readonly jenkinsAccess?: jenkinsAccess;
     readonly githubAutomationAppAccess?: string;
+    readonly githubEventsBucket: Bucket;
 }
 
 
@@ -43,15 +45,16 @@ export class OpenSearchDomainStack extends Stack {
     public readonly domain: Domain;
     public readonly props: OpenSearchStackProps;
     public readonly fullAccessRole: IRole;
-    public readonly openSearchLambdaRole: IRole;
+    public readonly openSearchMetricsLambdaRole: IRole;
+    public readonly openSearchS3EventsIndexLambdaRole: IRole;
     public readonly opensearchDomainConfig: OpenSearchDomainConfig;
 
     constructor(scope: Construct, id: string, props: OpenSearchStackProps) {
         super(scope, id);
         this.props = props;
 
 
-        this.openSearchLambdaRole = new Role(this, 'OpenSearchDomainLambdaRole', {
+        this.openSearchMetricsLambdaRole = new Role(this, 'OpenSearchDomainLambdaRole', {
             assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
             description: "OpenSearch Metrics Lambda Execution Role",
             roleName: "OpenSearchLambdaRole",
@@ -76,6 +79,41 @@ export class OpenSearchDomainStack extends Stack {
             ]
         });
 
+        this.openSearchS3EventsIndexLambdaRole = new Role(this, 'OpenSearchS3EventIndexLambdaRole', {
+            assumedBy: new ServicePrincipal('lambda.amazonaws.com'),
+            description: "OpenSearch Metrics S3 Event Index Lambda Execution Role",
+            roleName: "OpenSearchS3EventIndexLambdaRole",
+            inlinePolicies: {
+                "opensearchAssumeRolePolicy": new PolicyDocument({
+                    statements: [
+                        new PolicyStatement({
+                            effect: Effect.ALLOW,
+                            actions: ["sts:AssumeRole"],
+                            resources: [`arn:aws:iam::${props.account}:role/OpenSearchFullAccessRole`],
+                            conditions: {
+                                StringEquals: { 'aws:PrincipalAccount': props.account, 'aws:RequestedRegion': props.region, },
+                            }
+                        })
+                    ]
+                }),
+                "opensearchReadS3EventsPolicy": new PolicyDocument({
+                    statements: [
+                        new PolicyStatement({
+                            effect: Effect.ALLOW,
+                            actions: ["s3:GetObject",
+                                "s3:ListBucket"],
+                            resources: [props.githubEventsBucket.bucketArn,
+                                `${props.githubEventsBucket.bucketArn}/*`],
+                        })
+                    ]
+                })
+            },
+            managedPolicies: [
+                ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole'),
+                ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole'),
+            ]
+        });
+
         this.opensearchDomainConfig = {
             domainName: 'opensearch-health',
             ebsOptions: {
@@ -85,7 +123,7 @@ export class OpenSearchDomainStack extends Stack {
 
         const domainArn = `arn:aws:es:${props.region}:${props.account}:domain/${this.opensearchDomainConfig.domainName}/*`;
 
-        const secureRolesList = [this.openSearchLambdaRole]
+        const secureRolesList = [this.openSearchMetricsLambdaRole, this.openSearchS3EventsIndexLambdaRole]
         this.fullAccessRole = new Role(this, 'OpenSearchFullAccessRole', {
             assumedBy: new CompositePrincipal(...secureRolesList.map((role) => new ArnPrincipal(role.roleArn))),
             description: "Master role for OpenSearch full access",

infrastructure/lib/stacks/s3EventIndexWorkflow.ts

@@ -0,0 +1,83 @@
+/**
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import { Duration, Stack, StackProps } from "aws-cdk-lib";
+import { Rule, Schedule } from "aws-cdk-lib/aws-events";
+import { SfnStateMachine } from "aws-cdk-lib/aws-events-targets";
+import {JsonPath, StateMachine, TaskInput} from "aws-cdk-lib/aws-stepfunctions";
+import { LambdaInvoke } from "aws-cdk-lib/aws-stepfunctions-tasks";
+import { Construct } from 'constructs';
+import { OpenSearchLambda } from "../constructs/lambda";
+import { OpenSearchDomainStack } from "./opensearch";
+import { VpcStack } from "./vpc";
+import {Bucket} from "aws-cdk-lib/aws-s3";
+
+export interface OpenSearchS3EventIndexWorkflowStackProps extends StackProps {
+    readonly region: string;
+    readonly opensearchDomainStack: OpenSearchDomainStack;
+    readonly vpcStack: VpcStack;
+    readonly lambdaPackage: string;
+    readonly githubEventsBucket: Bucket;
+}
+
+export interface WorkflowComponent {
+    opensearchS3EventIndexWorkflowStateMachineName: string
+}
+export class OpenSearchS3EventIndexWorkflowStack extends Stack {
+    public readonly workflowComponent: WorkflowComponent;
+    constructor(scope: Construct, id: string, props: OpenSearchS3EventIndexWorkflowStackProps) {
+        super(scope, id, props);
+
+        const s3EventIndexTask = this.createS3EventIndexTask(this,
+            props.region,
+            props.opensearchDomainStack,
+            props.vpcStack,
+            props.lambdaPackage,
+            props.githubEventsBucket);
+
+        const opensearchS3EventIndexWorkflow = new StateMachine(this, 'OpenSearchS3EventIndexWorkflow', {
+            definition: s3EventIndexTask,
+            timeout: Duration.minutes(15),
+            stateMachineName: 'OpenSearchS3EventIndexWorkflow'
+        })
+
+        new Rule(this, 'OpenSearchS3EventIndexWorkflow-Every-Day', {
+            schedule: Schedule.expression('cron(0 0 * * ? *)'),
+            targets: [new SfnStateMachine(opensearchS3EventIndexWorkflow)],
+        });
+
+        this.workflowComponent = {
+            opensearchS3EventIndexWorkflowStateMachineName: opensearchS3EventIndexWorkflow.stateMachineName
+        }
+    }
+
+    private createS3EventIndexTask(scope: Construct, region: string, opensearchDomainStack: OpenSearchDomainStack, vpcStack: VpcStack, lambdaPackage: string, githubEventsBucket: Bucket) {
+        const openSearchDomain = opensearchDomainStack.domain;
+        const s3EventIndexLambda = new OpenSearchLambda(this, "OpenSearchMetricsS3EventIndexLambdaFunction", {
+            lambdaNameBase: "OpenSearchMetricsS3EventIndex",
+            handler: "org.opensearchmetrics.lambda.GithubEventsLambda",
+            lambdaZipPath: `../../../build/distributions/${lambdaPackage}`,
+            vpc: vpcStack.vpc,
+            securityGroup: vpcStack.securityGroup,
+            role: opensearchDomainStack.openSearchS3EventsIndexLambdaRole,
+            environment: {
+                S3_BUCKET_REGION: region,
+                EVENT_BUCKET_NAME: githubEventsBucket.bucketName,
+                OPENSEARCH_DOMAIN_ENDPOINT: openSearchDomain.domainEndpoint,
+                OPENSEARCH_DOMAIN_REGION: openSearchDomain.env.region,
+                OPENSEARCH_DOMAIN_ROLE: opensearchDomainStack.fullAccessRole.roleArn,
+            }
+        }).lambda;
+        return new LambdaInvoke(scope, 'S3 Event Index Lambda', {
+            lambdaFunction: s3EventIndexLambda,
+            resultPath: JsonPath.DISCARD,
+            payload: TaskInput.fromJsonPathAt("$"),
+            timeout: Duration.minutes(15)
+        }).addRetry();
+    }
+}

infrastructure/test/workflow-stack.test.ts → infrastructure/test/metrics-workflow-stack.test.ts

@@ -13,25 +13,30 @@ import Project from "../lib/enums/project";
 import { OpenSearchDomainStack } from "../lib/stacks/opensearch";
 import { VpcStack } from "../lib/stacks/vpc";
 import { ArnPrincipal } from "aws-cdk-lib/aws-iam";
+import {OpenSearchS3} from "../lib/stacks/s3";
 
-test('Workflow Stack Test', () => {
+test('Metrics Workflow Stack Test', () => {
     const app = new App();
-    const vpcStack = new VpcStack(app, 'Test-OpenSearchHealth-VPC', {})
+    const vpcStack = new VpcStack(app, 'Test-OpenSearchHealth-VPC', {});
+    const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3");
+    const openSearchDomainStack = new OpenSearchDomainStack(app, 'OpenSearchHealth-OpenSearch', {
+        region: "us-east-1",
+        account: "test-account",
+        vpcStack: new VpcStack(app, 'OpenSearchHealth-VPC', {}),
+        enableNginxCognito: true,
+        jenkinsAccess: {
+            jenkinsAccountRoles: [
+                new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
+                new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
+            ]
+        },
+        githubAutomationAppAccess: "sample-role-arn",
+        githubEventsBucket: s3Stack.bucket,
+    });
     const OpenSearchMetricsWorkflow = new OpenSearchMetricsWorkflowStack(app, 'Test-OpenSearchMetrics-Workflow', {
-        opensearchDomainStack: new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', {
-            region: "us-east-1",
-            account: "test-account",
-            vpcStack: vpcStack,
-            enableNginxCognito: true,
-            jenkinsAccess: {
-                jenkinsAccountRoles: [
-                    new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
-                    new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
-                ]
-            }
-        }),
+        opensearchDomainStack: openSearchDomainStack,
         vpcStack: vpcStack,
-        lambdaPackage: Project.LAMBDA_PACKAGE
+        lambdaPackage: Project.LAMBDA_PACKAGE,
     });
     const template = Template.fromStack(OpenSearchMetricsWorkflow);
     template.resourceCountIs('AWS::IAM::Role', 2);
@@ -69,4 +74,4 @@ test('Workflow Stack Test', () => {
         },
         "StateMachineName": "OpenSearchMetricsWorkflow"
     });
-});
+});