diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 49b026f..ac77013 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -26,6 +26,7 @@ So you want to contribute code to this project? Excellent! We're glad you're her - `cdk deploy OpenSearchMetrics-GitHubWorkflowMonitor-Alarms`: Creates the Alarms to Monitor the Critical GitHub CI workflows by the GitHub Automation App. - `cdk deploy OpenSearchMetrics-GitHubAutomationApp`: Create the resources which launches the [GitHub Automation App](https://github.com/opensearch-project/automation-app). Listens to GitHub events and index the data to Metrics cluster. - `cdk deploy OpenSearchMetrics-GitHubAutomationAppEvents-S3`: Creates the S3 Bucket for the [GitHub Automation App](https://github.com/opensearch-project/automation-app) to store OpenSearch Project GitHub Events. + - `cdk deploy OpenSearchS3EventIndex-Workflow`: Creates the Lambda and Step Function to index the GitHub Events stored in the S3 Bucket to the Metrics cluster. ### Forking and Cloning diff --git a/build.gradle b/build.gradle index 3b240a2..32da440 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,8 @@ dependencies { implementation 'com.amazonaws:aws-java-sdk-secretsmanager:1.12.671' + implementation 'software.amazon.awssdk:s3:2.28.17' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' diff --git a/infrastructure/lib/infrastructure-stack.ts b/infrastructure/lib/infrastructure-stack.ts index 7171ff3..c081e91 100644 --- a/infrastructure/lib/infrastructure-stack.ts +++ b/infrastructure/lib/infrastructure-stack.ts @@ -22,6 +22,7 @@ import { OpenSearchMetricsSecretsStack } from "./stacks/secrets"; import { GitHubAutomationApp } from "./stacks/gitHubAutomationApp"; import { OpenSearchS3 } from "./stacks/s3"; import { GitHubWorkflowMonitorAlarms } from "./stacks/gitHubWorkflowMonitorAlarms"; +import {OpenSearchS3EventIndexWorkflowStack} from "./stacks/s3EventIndexWorkflow"; export class InfrastructureStack extends Stack { constructor(scope: Construct, id: string, props?: StackProps) { @@ -75,15 +76,28 @@ export class InfrastructureStack extends Stack { new ArnPrincipal(Project.JENKINS_AGENT_ROLE) ] }, - githubAutomationAppAccess: gitHubAutomationApp.githubAppRole.roleArn + githubAutomationAppAccess: gitHubAutomationApp.githubAppRole.roleArn, + githubEventsBucket: openSearchEventsS3Bucket.bucket, }); // Create OpenSearch Metrics Lambda setup const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', { - opensearchDomainStack: openSearchDomainStack, vpcStack: vpcStack, lambdaPackage: Project.LAMBDA_PACKAGE + opensearchDomainStack: openSearchDomainStack, + vpcStack: vpcStack, + lambdaPackage: Project.LAMBDA_PACKAGE, }) openSearchMetricsWorkflowStack.node.addDependency(vpcStack, openSearchDomainStack); + // Create OpenSearch S3 Event Index Lambda setup + const openSearchS3EventIndexWorkflowStack = new OpenSearchS3EventIndexWorkflowStack(app, 'OpenSearchS3EventIndex-Workflow', { + region: Project.REGION, + opensearchDomainStack: openSearchDomainStack, + vpcStack: vpcStack, + lambdaPackage: Project.LAMBDA_PACKAGE, + githubEventsBucket: openSearchEventsS3Bucket.bucket + }) + openSearchS3EventIndexWorkflowStack.node.addDependency(vpcStack, openSearchDomainStack); + // Create Secret Manager for the metrics project const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", { secretName: 'metrics-creds' @@ -94,7 +108,10 @@ export class InfrastructureStack extends Stack { const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", { region: Project.REGION, account: Project.AWS_ACCOUNT, - workflowComponent: openSearchMetricsWorkflowStack.workflowComponent, + workflowComponent: { + opensearchMetricsWorkflowStateMachineName: openSearchMetricsWorkflowStack.workflowComponent.opensearchMetricsWorkflowStateMachineName, + opensearchS3EventIndexWorkflowStateMachineName: openSearchS3EventIndexWorkflowStack.workflowComponent.opensearchS3EventIndexWorkflowStateMachineName + }, lambdaPackage: Project.LAMBDA_PACKAGE, secrets: openSearchMetricsSecretsStack.secret, vpcStack: vpcStack diff --git a/infrastructure/lib/stacks/metricsWorkflow.ts b/infrastructure/lib/stacks/metricsWorkflow.ts index f18aa01..2c8e349 100644 --- a/infrastructure/lib/stacks/metricsWorkflow.ts +++ b/infrastructure/lib/stacks/metricsWorkflow.ts @@ -53,7 +53,7 @@ export class OpenSearchMetricsWorkflowStack extends Stack { } private createMetricsTask(scope: Construct, opensearchDomainStack: OpenSearchDomainStack, - vpcStack: VpcStack, lambdaPackage: string) { + vpcStack: VpcStack, lambdaPackage: string) { const openSearchDomain = opensearchDomainStack.domain; const metricsLambda = new OpenSearchLambda(scope, "OpenSearchMetricsLambdaFunction", { lambdaNameBase: "OpenSearchMetricsDashboards", @@ -61,7 +61,7 @@ export class OpenSearchMetricsWorkflowStack extends Stack { lambdaZipPath: `../../../build/distributions/${lambdaPackage}`, vpc: vpcStack.vpc, securityGroup: vpcStack.securityGroup, - role: opensearchDomainStack.openSearchLambdaRole, + role: opensearchDomainStack.openSearchMetricsLambdaRole, environment: { OPENSEARCH_DOMAIN_ENDPOINT: openSearchDomain.domainEndpoint, OPENSEARCH_DOMAIN_REGION: openSearchDomain.env.region, diff --git a/infrastructure/lib/stacks/monitoringDashboard.ts b/infrastructure/lib/stacks/monitoringDashboard.ts index 65a0ea9..1f72356 100644 --- a/infrastructure/lib/stacks/monitoringDashboard.ts +++ b/infrastructure/lib/stacks/monitoringDashboard.ts @@ -16,14 +16,13 @@ import { canarySns } from "../constructs/canarySns"; import { OpenSearchLambda } from "../constructs/lambda"; import { StepFunctionSns } from "../constructs/stepFunctionSns"; import Project from "../enums/project"; -import { WorkflowComponent } from "./metricsWorkflow"; import { VpcStack } from "./vpc"; interface OpenSearchMetricsMonitoringStackProps extends StackProps { readonly region: string; readonly account: string; - readonly workflowComponent: WorkflowComponent; + readonly workflowComponent: {[component: string]: string}; readonly lambdaPackage: string; readonly secrets: Secret; readonly vpcStack: VpcStack; @@ -70,6 +69,7 @@ export class OpenSearchMetricsMonitoringStack extends Stack { private snsMonitorStepFunctionExecutionsFailed(): void { const stepFunctionSnsAlarms = [ { alertName: 'StepFunction_execution_errors_MetricsWorkflow', stateMachineName: this.props.workflowComponent.opensearchMetricsWorkflowStateMachineName }, + { alertName: 'StepFunction_execution_errors_S3EventIndexWorkflow', stateMachineName: this.props.workflowComponent.opensearchS3EventIndexWorkflowStateMachineName }, ]; new StepFunctionSns(this, "SnsMonitors-StepFunctionExecutionsFailed", { diff --git a/infrastructure/lib/stacks/opensearch.ts b/infrastructure/lib/stacks/opensearch.ts index 769f78e..6fe140e 100644 --- a/infrastructure/lib/stacks/opensearch.ts +++ b/infrastructure/lib/stacks/opensearch.ts @@ -15,6 +15,7 @@ import { OpenSearchMetricsNginxCognito } from "../constructs/opensearchNginxProx import Project from "../enums/project"; import { OpenSearchHealthRoute53 } from "./route53"; import { VpcStack } from "./vpc"; +import {Bucket} from "aws-cdk-lib/aws-s3"; export interface OpenSearchStackProps { @@ -24,6 +25,7 @@ export interface OpenSearchStackProps { readonly enableNginxCognito: boolean; readonly jenkinsAccess?: jenkinsAccess; readonly githubAutomationAppAccess?: string; + readonly githubEventsBucket: Bucket; } @@ -43,7 +45,8 @@ export class OpenSearchDomainStack extends Stack { public readonly domain: Domain; public readonly props: OpenSearchStackProps; public readonly fullAccessRole: IRole; - public readonly openSearchLambdaRole: IRole; + public readonly openSearchMetricsLambdaRole: IRole; + public readonly openSearchS3EventsIndexLambdaRole: IRole; public readonly opensearchDomainConfig: OpenSearchDomainConfig; constructor(scope: Construct, id: string, props: OpenSearchStackProps) { @@ -51,7 +54,7 @@ export class OpenSearchDomainStack extends Stack { this.props = props; - this.openSearchLambdaRole = new Role(this, 'OpenSearchDomainLambdaRole', { + this.openSearchMetricsLambdaRole = new Role(this, 'OpenSearchDomainLambdaRole', { assumedBy: new ServicePrincipal('lambda.amazonaws.com'), description: "OpenSearch Metrics Lambda Execution Role", roleName: "OpenSearchLambdaRole", @@ -76,6 +79,41 @@ export class OpenSearchDomainStack extends Stack { ] }); + this.openSearchS3EventsIndexLambdaRole = new Role(this, 'OpenSearchS3EventIndexLambdaRole', { + assumedBy: new ServicePrincipal('lambda.amazonaws.com'), + description: "OpenSearch Metrics S3 Event Index Lambda Execution Role", + roleName: "OpenSearchS3EventIndexLambdaRole", + inlinePolicies: { + "opensearchAssumeRolePolicy": new PolicyDocument({ + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["sts:AssumeRole"], + resources: [`arn:aws:iam::${props.account}:role/OpenSearchFullAccessRole`], + conditions: { + StringEquals: { 'aws:PrincipalAccount': props.account, 'aws:RequestedRegion': props.region, }, + } + }) + ] + }), + "opensearchReadS3EventsPolicy": new PolicyDocument({ + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["s3:GetObject", + "s3:ListBucket"], + resources: [props.githubEventsBucket.bucketArn, + `${props.githubEventsBucket.bucketArn}/*`], + }) + ] + }) + }, + managedPolicies: [ + ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole'), + ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole'), + ] + }); + this.opensearchDomainConfig = { domainName: 'opensearch-health', ebsOptions: { @@ -85,7 +123,7 @@ export class OpenSearchDomainStack extends Stack { const domainArn = `arn:aws:es:${props.region}:${props.account}:domain/${this.opensearchDomainConfig.domainName}/*`; - const secureRolesList = [this.openSearchLambdaRole] + const secureRolesList = [this.openSearchMetricsLambdaRole, this.openSearchS3EventsIndexLambdaRole] this.fullAccessRole = new Role(this, 'OpenSearchFullAccessRole', { assumedBy: new CompositePrincipal(...secureRolesList.map((role) => new ArnPrincipal(role.roleArn))), description: "Master role for OpenSearch full access", diff --git a/infrastructure/lib/stacks/s3EventIndexWorkflow.ts b/infrastructure/lib/stacks/s3EventIndexWorkflow.ts new file mode 100644 index 0000000..cebe001 --- /dev/null +++ b/infrastructure/lib/stacks/s3EventIndexWorkflow.ts @@ -0,0 +1,83 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { Duration, Stack, StackProps } from "aws-cdk-lib"; +import { Rule, Schedule } from "aws-cdk-lib/aws-events"; +import { SfnStateMachine } from "aws-cdk-lib/aws-events-targets"; +import {JsonPath, StateMachine, TaskInput} from "aws-cdk-lib/aws-stepfunctions"; +import { LambdaInvoke } from "aws-cdk-lib/aws-stepfunctions-tasks"; +import { Construct } from 'constructs'; +import { OpenSearchLambda } from "../constructs/lambda"; +import { OpenSearchDomainStack } from "./opensearch"; +import { VpcStack } from "./vpc"; +import {Bucket} from "aws-cdk-lib/aws-s3"; + +export interface OpenSearchS3EventIndexWorkflowStackProps extends StackProps { + readonly region: string; + readonly opensearchDomainStack: OpenSearchDomainStack; + readonly vpcStack: VpcStack; + readonly lambdaPackage: string; + readonly githubEventsBucket: Bucket; +} + +export interface WorkflowComponent { + opensearchS3EventIndexWorkflowStateMachineName: string +} +export class OpenSearchS3EventIndexWorkflowStack extends Stack { + public readonly workflowComponent: WorkflowComponent; + constructor(scope: Construct, id: string, props: OpenSearchS3EventIndexWorkflowStackProps) { + super(scope, id, props); + + const s3EventIndexTask = this.createS3EventIndexTask(this, + props.region, + props.opensearchDomainStack, + props.vpcStack, + props.lambdaPackage, + props.githubEventsBucket); + + const opensearchS3EventIndexWorkflow = new StateMachine(this, 'OpenSearchS3EventIndexWorkflow', { + definition: s3EventIndexTask, + timeout: Duration.minutes(15), + stateMachineName: 'OpenSearchS3EventIndexWorkflow' + }) + + new Rule(this, 'OpenSearchS3EventIndexWorkflow-Every-Day', { + schedule: Schedule.expression('cron(0 0 * * ? *)'), + targets: [new SfnStateMachine(opensearchS3EventIndexWorkflow)], + }); + + this.workflowComponent = { + opensearchS3EventIndexWorkflowStateMachineName: opensearchS3EventIndexWorkflow.stateMachineName + } + } + + private createS3EventIndexTask(scope: Construct, region: string, opensearchDomainStack: OpenSearchDomainStack, vpcStack: VpcStack, lambdaPackage: string, githubEventsBucket: Bucket) { + const openSearchDomain = opensearchDomainStack.domain; + const s3EventIndexLambda = new OpenSearchLambda(this, "OpenSearchMetricsS3EventIndexLambdaFunction", { + lambdaNameBase: "OpenSearchMetricsS3EventIndex", + handler: "org.opensearchmetrics.lambda.GithubEventsLambda", + lambdaZipPath: `../../../build/distributions/${lambdaPackage}`, + vpc: vpcStack.vpc, + securityGroup: vpcStack.securityGroup, + role: opensearchDomainStack.openSearchS3EventsIndexLambdaRole, + environment: { + S3_BUCKET_REGION: region, + EVENT_BUCKET_NAME: githubEventsBucket.bucketName, + OPENSEARCH_DOMAIN_ENDPOINT: openSearchDomain.domainEndpoint, + OPENSEARCH_DOMAIN_REGION: openSearchDomain.env.region, + OPENSEARCH_DOMAIN_ROLE: opensearchDomainStack.fullAccessRole.roleArn, + } + }).lambda; + return new LambdaInvoke(scope, 'S3 Event Index Lambda', { + lambdaFunction: s3EventIndexLambda, + resultPath: JsonPath.DISCARD, + payload: TaskInput.fromJsonPathAt("$"), + timeout: Duration.minutes(15) + }).addRetry(); + } +} diff --git a/infrastructure/test/workflow-stack.test.ts b/infrastructure/test/metrics-workflow-stack.test.ts similarity index 74% rename from infrastructure/test/workflow-stack.test.ts rename to infrastructure/test/metrics-workflow-stack.test.ts index 83319cc..9f8f378 100644 --- a/infrastructure/test/workflow-stack.test.ts +++ b/infrastructure/test/metrics-workflow-stack.test.ts @@ -13,25 +13,30 @@ import Project from "../lib/enums/project"; import { OpenSearchDomainStack } from "../lib/stacks/opensearch"; import { VpcStack } from "../lib/stacks/vpc"; import { ArnPrincipal } from "aws-cdk-lib/aws-iam"; +import {OpenSearchS3} from "../lib/stacks/s3"; -test('Workflow Stack Test', () => { +test('Metrics Workflow Stack Test', () => { const app = new App(); - const vpcStack = new VpcStack(app, 'Test-OpenSearchHealth-VPC', {}) + const vpcStack = new VpcStack(app, 'Test-OpenSearchHealth-VPC', {}); + const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3"); + const openSearchDomainStack = new OpenSearchDomainStack(app, 'OpenSearchHealth-OpenSearch', { + region: "us-east-1", + account: "test-account", + vpcStack: new VpcStack(app, 'OpenSearchHealth-VPC', {}), + enableNginxCognito: true, + jenkinsAccess: { + jenkinsAccountRoles: [ + new ArnPrincipal(Project.JENKINS_MASTER_ROLE), + new ArnPrincipal(Project.JENKINS_AGENT_ROLE) + ] + }, + githubAutomationAppAccess: "sample-role-arn", + githubEventsBucket: s3Stack.bucket, + }); const OpenSearchMetricsWorkflow = new OpenSearchMetricsWorkflowStack(app, 'Test-OpenSearchMetrics-Workflow', { - opensearchDomainStack: new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', { - region: "us-east-1", - account: "test-account", - vpcStack: vpcStack, - enableNginxCognito: true, - jenkinsAccess: { - jenkinsAccountRoles: [ - new ArnPrincipal(Project.JENKINS_MASTER_ROLE), - new ArnPrincipal(Project.JENKINS_AGENT_ROLE) - ] - } - }), + opensearchDomainStack: openSearchDomainStack, vpcStack: vpcStack, - lambdaPackage: Project.LAMBDA_PACKAGE + lambdaPackage: Project.LAMBDA_PACKAGE, }); const template = Template.fromStack(OpenSearchMetricsWorkflow); template.resourceCountIs('AWS::IAM::Role', 2); @@ -69,4 +74,4 @@ test('Workflow Stack Test', () => { }, "StateMachineName": "OpenSearchMetricsWorkflow" }); -}); \ No newline at end of file +}); diff --git a/infrastructure/test/monitoring-stack.test.ts b/infrastructure/test/monitoring-stack.test.ts index a8544ab..ed770a0 100644 --- a/infrastructure/test/monitoring-stack.test.ts +++ b/infrastructure/test/monitoring-stack.test.ts @@ -15,25 +15,37 @@ import { VpcStack } from "../lib/stacks/vpc"; import { ArnPrincipal } from "aws-cdk-lib/aws-iam"; import { OpenSearchMetricsMonitoringStack } from "../lib/stacks/monitoringDashboard"; import { OpenSearchMetricsSecretsStack } from "../lib/stacks/secrets"; +import {OpenSearchS3} from "../lib/stacks/s3"; +import {OpenSearchS3EventIndexWorkflowStack} from "../lib/stacks/s3EventIndexWorkflow"; test('Monitoring Stack Test', () => { const app = new App(); const vpcStack = new VpcStack(app, 'OpenSearchHealth-VPC', {}); - const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', { - opensearchDomainStack: new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', { - region: "us-east-1", - account: "test-account", - vpcStack: vpcStack, - enableNginxCognito: true, - jenkinsAccess: { - jenkinsAccountRoles: [ - new ArnPrincipal(Project.JENKINS_MASTER_ROLE), - new ArnPrincipal(Project.JENKINS_AGENT_ROLE) - ] - } - }), + const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3"); + const opensearchDomainStack = new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', { + region: "us-east-1", + account: "test-account", + vpcStack: vpcStack, + enableNginxCognito: true, + jenkinsAccess: { + jenkinsAccountRoles: [ + new ArnPrincipal(Project.JENKINS_MASTER_ROLE), + new ArnPrincipal(Project.JENKINS_AGENT_ROLE) + ] + }, + githubEventsBucket: s3Stack.bucket + }); + const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'Test-OpenSearchMetrics-Workflow', { + opensearchDomainStack: opensearchDomainStack, vpcStack: vpcStack, - lambdaPackage: Project.LAMBDA_PACKAGE + lambdaPackage: Project.LAMBDA_PACKAGE, + }); + const openSearchS3EventIndexWorkflowStack = new OpenSearchS3EventIndexWorkflowStack(app, 'Test-OpenSearchS3EventIndex-Workflow', { + region: Project.REGION, + opensearchDomainStack: opensearchDomainStack, + vpcStack: vpcStack, + lambdaPackage: Project.LAMBDA_PACKAGE, + githubEventsBucket: s3Stack.bucket }); const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", { secretName: 'metrics-creds' @@ -41,7 +53,10 @@ test('Monitoring Stack Test', () => { const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", { region: Project.REGION, account: Project.AWS_ACCOUNT, - workflowComponent: openSearchMetricsWorkflowStack.workflowComponent, + workflowComponent: { + opensearchMetricsWorkflowStateMachineName: openSearchMetricsWorkflowStack.workflowComponent.opensearchMetricsWorkflowStateMachineName, + opensearchS3EventIndexWorkflowStateMachineName: openSearchS3EventIndexWorkflowStack.workflowComponent.opensearchS3EventIndexWorkflowStateMachineName + }, lambdaPackage: Project.LAMBDA_PACKAGE, secrets: openSearchMetricsSecretsStack.secret, vpcStack: vpcStack @@ -49,7 +64,7 @@ test('Monitoring Stack Test', () => { const template = Template.fromStack(openSearchMetricsMonitoringStack); template.resourceCountIs('AWS::IAM::Role', 2); template.resourceCountIs('AWS::IAM::Policy', 1); - template.resourceCountIs('AWS::CloudWatch::Alarm', 2); + template.resourceCountIs('AWS::CloudWatch::Alarm', 3); template.resourceCountIs('AWS::SNS::Topic', 2); template.resourceCountIs('AWS::Synthetics::Canary', 1); template.hasResourceProperties('AWS::IAM::Role', { @@ -141,7 +156,41 @@ test('Monitoring Stack Test', () => { [ "arn:aws:states:::stateMachine:", { - "Fn::ImportValue": "OpenSearchMetrics-Workflow:ExportsOutputFnGetAttOpenSearchMetricsWorkflowDB4D4CB1NameE4E75A02" + "Fn::ImportValue": "Test-OpenSearchMetrics-Workflow:ExportsOutputFnGetAttOpenSearchMetricsWorkflowDB4D4CB1NameE4E75A02" + } + ] + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ExecutionsFailed", + "Namespace": "AWS/States", + "Period": 300, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching" + }); + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + "AlarmActions": [ + { + "Ref": "SnsMonitorsStepFunctionExecutionsFailedOpenSearchMetricsAlarmStepFunctionExecutionsFailed0B259DBC" + } + ], + "AlarmDescription": "Detect SF execution failure", + "AlarmName": "StepFunction_execution_errors_S3EventIndexWorkflow", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "DatapointsToAlarm": 1, + "Dimensions": [ + { + "Name": "StateMachineArn", + "Value": { + "Fn::Join": [ + "", + [ + "arn:aws:states:::stateMachine:", + { + "Fn::ImportValue": "Test-OpenSearchS3EventIndex-Workflow:ExportsOutputFnGetAttOpenSearchS3EventIndexWorkflow0C74BA9FName074F0965" } ] ] diff --git a/infrastructure/test/nginx.test.ts b/infrastructure/test/nginx.test.ts index 7b71eda..3283248 100644 --- a/infrastructure/test/nginx.test.ts +++ b/infrastructure/test/nginx.test.ts @@ -14,10 +14,12 @@ import Project from "../lib/enums/project"; import { OpenSearchDomainStack } from "../lib/stacks/opensearch"; import { ArnPrincipal } from "aws-cdk-lib/aws-iam"; import { OpenSearchHealthRoute53 } from "../lib/stacks/route53"; +import {OpenSearchS3} from "../lib/stacks/s3"; test('OpenSearchMetricsNginxReadonly Stack Test', () => { const app = new App(); const vpcStack = new VpcStack(app, "OpenSearchHealth-VPC", {}); + const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3"); const openSearchDomainStack = new OpenSearchDomainStack(app, "OpenSearchHealth-OpenSearch", { region: Project.REGION, account: Project.AWS_ACCOUNT, @@ -28,7 +30,8 @@ test('OpenSearchMetricsNginxReadonly Stack Test', () => { new ArnPrincipal(Project.JENKINS_MASTER_ROLE), new ArnPrincipal(Project.JENKINS_AGENT_ROLE) ] - } + }, + githubEventsBucket: s3Stack.bucket }); const metricsHostedZone = new OpenSearchHealthRoute53(app, "OpenSearchMetrics-HostedZone", { hostedZone: Project.METRICS_HOSTED_ZONE, diff --git a/infrastructure/test/opensearch-stack.test.ts b/infrastructure/test/opensearch-stack.test.ts index 49cd35d..16e6c9d 100644 --- a/infrastructure/test/opensearch-stack.test.ts +++ b/infrastructure/test/opensearch-stack.test.ts @@ -12,9 +12,11 @@ import { OpenSearchDomainStack } from "../lib/stacks/opensearch"; import { ArnPrincipal } from "aws-cdk-lib/aws-iam"; import Project from "../lib/enums/project"; import { VpcStack } from "../lib/stacks/vpc"; +import {OpenSearchS3} from "../lib/stacks/s3"; test('OpenSearchDomain Stack Test', () => { const app = new App(); + const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3"); const openSearchDomainStack = new OpenSearchDomainStack(app, 'OpenSearchHealth-OpenSearch', { region: "us-east-1", account: "test-account", @@ -26,10 +28,11 @@ test('OpenSearchDomain Stack Test', () => { new ArnPrincipal(Project.JENKINS_AGENT_ROLE) ] }, - githubAutomationAppAccess: "sample-role-arn" + githubAutomationAppAccess: "sample-role-arn", + githubEventsBucket: s3Stack.bucket, }); const openSearchDomainStackTemplate = Template.fromStack(openSearchDomainStack); - openSearchDomainStackTemplate.resourceCountIs('AWS::IAM::Role', 8); + openSearchDomainStackTemplate.resourceCountIs('AWS::IAM::Role', 9); openSearchDomainStackTemplate.resourceCountIs('AWS::Cognito::UserPool', 1); openSearchDomainStackTemplate.resourceCountIs('AWS::Cognito::UserPoolGroup', 1); openSearchDomainStackTemplate.resourceCountIs('AWS::IAM::Policy', 4); @@ -57,6 +60,100 @@ test('OpenSearchDomain Stack Test', () => { "Version": "2012-10-17" } }); + openSearchDomainStackTemplate.hasResourceProperties('AWS::IAM::Role', { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Description": "OpenSearch Metrics S3 Event Index Lambda Execution Role", + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + ] + ] + } + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "aws:PrincipalAccount": "test-account", + "aws:RequestedRegion": "us-east-1" + } + }, + "Effect": "Allow", + "Resource": "arn:aws:iam::test-account:role/OpenSearchFullAccessRole" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "opensearchAssumeRolePolicy" + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject", + "s3:ListBucket" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::ImportValue": "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3:ExportsOutputFnGetAttOpenSearchS3Bucket2ED683CCArnC283B682" + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::ImportValue": "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3:ExportsOutputFnGetAttOpenSearchS3Bucket2ED683CCArnC283B682" + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "opensearchReadS3EventsPolicy" + } + ], + "RoleName": "OpenSearchS3EventIndexLambdaRole" + }); openSearchDomainStackTemplate.resourceCountIs('AWS::Cognito::UserPoolGroup', 1); openSearchDomainStackTemplate.hasResourceProperties('AWS::Cognito::UserPoolGroup', { "GroupName": "opensearch-admin-group", diff --git a/infrastructure/test/s3-event-index-workflow-stack.test.ts b/infrastructure/test/s3-event-index-workflow-stack.test.ts new file mode 100644 index 0000000..5f26a30 --- /dev/null +++ b/infrastructure/test/s3-event-index-workflow-stack.test.ts @@ -0,0 +1,79 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { App } from "aws-cdk-lib"; +import { Template } from "aws-cdk-lib/assertions"; +import Project from "../lib/enums/project"; +import { OpenSearchDomainStack } from "../lib/stacks/opensearch"; +import { VpcStack } from "../lib/stacks/vpc"; +import { ArnPrincipal } from "aws-cdk-lib/aws-iam"; +import {OpenSearchS3} from "../lib/stacks/s3"; +import {OpenSearchS3EventIndexWorkflowStack} from "../lib/stacks/s3EventIndexWorkflow"; + +test('S3 Event Index Workflow Stack Test', () => { + const app = new App(); + const vpcStack = new VpcStack(app, 'Test-OpenSearchHealth-VPC', {}); + const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3"); + const openSearchDomainStack = new OpenSearchDomainStack(app, 'OpenSearchHealth-OpenSearch', { + region: "us-east-1", + account: "test-account", + vpcStack: new VpcStack(app, 'OpenSearchHealth-VPC', {}), + enableNginxCognito: true, + jenkinsAccess: { + jenkinsAccountRoles: [ + new ArnPrincipal(Project.JENKINS_MASTER_ROLE), + new ArnPrincipal(Project.JENKINS_AGENT_ROLE) + ] + }, + githubAutomationAppAccess: "sample-role-arn", + githubEventsBucket: s3Stack.bucket, + }); + const OpenSearchS3EventIndexWorkflow = new OpenSearchS3EventIndexWorkflowStack(app, 'Test-OpenSearchS3EventIndex-Workflow', { + region: Project.REGION, + opensearchDomainStack: openSearchDomainStack, + vpcStack: vpcStack, + lambdaPackage: Project.LAMBDA_PACKAGE, + githubEventsBucket: s3Stack.bucket + }); + const template = Template.fromStack(OpenSearchS3EventIndexWorkflow); + template.resourceCountIs('AWS::IAM::Role', 2); + template.resourceCountIs('AWS::Lambda::Function', 1); + template.hasResourceProperties('AWS::Lambda::Function', { + "FunctionName": "OpenSearchMetricsS3EventIndexLambda", + "Handler": "org.opensearchmetrics.lambda.GithubEventsLambda" + }); + template.resourceCountIs('AWS::StepFunctions::StateMachine', 1); + template.hasResourceProperties('AWS::StepFunctions::StateMachine', { + "DefinitionString": { + "Fn::Join": [ + "", + [ + "{\"StartAt\":\"S3 Event Index Lambda\",\"States\":{\"S3 Event Index Lambda\":{\"End\":true,\"Retry\":[{\"ErrorEquals\":[\"Lambda.ClientExecutionTimeoutException\",\"Lambda.ServiceException\",\"Lambda.AWSLambdaException\",\"Lambda.SdkClientException\"],\"IntervalSeconds\":2,\"MaxAttempts\":6,\"BackoffRate\":2},{\"ErrorEquals\":[\"States.ALL\"]}],\"Type\":\"Task\",\"TimeoutSeconds\":900,\"ResultPath\":null,\"Resource\":\"arn:", + { + "Ref": "AWS::Partition" + }, + ":states:::lambda:invoke\",\"Parameters\":{\"FunctionName\":\"", + { + "Fn::GetAtt": [ + "OpenSearchMetricsS3EventIndexLambda6B9F9C68", + "Arn" + ] + }, + "\",\"Payload.$\":\"$\"}}},\"TimeoutSeconds\":900}" + ] + ] + }, + "RoleArn": { + "Fn::GetAtt": [ + "OpenSearchS3EventIndexWorkflowRoleE2712A63", + "Arn" + ] + }, + "StateMachineName": "OpenSearchS3EventIndexWorkflow" + }); +}); diff --git a/src/main/java/org/opensearchmetrics/dagger/CommonModule.java b/src/main/java/org/opensearchmetrics/dagger/CommonModule.java index 2328685..0929b17 100644 --- a/src/main/java/org/opensearchmetrics/dagger/CommonModule.java +++ b/src/main/java/org/opensearchmetrics/dagger/CommonModule.java @@ -1,5 +1,6 @@ package org.opensearchmetrics.dagger; +import org.opensearchmetrics.util.S3Util; import org.opensearchmetrics.util.SecretsManagerUtil; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -21,8 +22,10 @@ import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.signer.Aws4Signer; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.s3.S3Client; import javax.inject.Singleton; @@ -34,6 +37,9 @@ public class CommonModule { private static final String OPENSEARCH_DOMAIN_ROLE = "OPENSEARCH_DOMAIN_ROLE"; private static final String ROLE_SESSION_NAME = "OpenSearchHealth"; private static final String SECRETS_MANAGER_REGION = "SECRETS_MANAGER_REGION"; + private static final String S3_BUCKET_REGION = "S3_BUCKET_REGION"; + private static final String EVENT_BUCKET_NAME = "EVENT_BUCKET_NAME"; + @Singleton @Provides @@ -56,6 +62,21 @@ private AwsCredentialsProvider stsAssumeRoleCredentialProvider(String roleSessio return credentialsProvider; } + @Singleton + @Provides + public S3Client getS3Client() { + final String region = System.getenv(S3_BUCKET_REGION); + return S3Client.builder() + .region(Region.of(region)) + .build(); + } + + @Singleton + @Provides + public S3Util getS3Util(S3Client s3Client) { + return new S3Util(s3Client, System.getenv(EVENT_BUCKET_NAME)); + } + @Singleton @Provides public RestHighLevelClient getOpenSearchHLClient() { diff --git a/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java b/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java index e214b88..c685ad3 100644 --- a/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java +++ b/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java @@ -7,6 +7,7 @@ import org.opensearchmetrics.metrics.label.LabelMetrics; import org.opensearchmetrics.util.OpenSearchUtil; import org.opensearchmetrics.util.SecretsManagerUtil; +import org.opensearchmetrics.util.S3Util; import javax.inject.Named; import javax.inject.Singleton; @@ -17,6 +18,8 @@ public interface ServiceComponent { ObjectMapper getObjectMapper(); + S3Util getS3Util(); + OpenSearchUtil getOpenSearchUtil(); MetricsCalculation getMetricsCalculation(); diff --git a/src/main/java/org/opensearchmetrics/lambda/GithubEventsLambda.java b/src/main/java/org/opensearchmetrics/lambda/GithubEventsLambda.java new file mode 100644 index 0000000..3879c50 --- /dev/null +++ b/src/main/java/org/opensearchmetrics/lambda/GithubEventsLambda.java @@ -0,0 +1,97 @@ +package org.opensearchmetrics.lambda; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.opensearchmetrics.dagger.DaggerServiceComponent; +import org.opensearchmetrics.dagger.ServiceComponent; +import org.opensearchmetrics.metrics.events.GithubEvents; +import org.opensearchmetrics.model.event.EventData; +import org.opensearchmetrics.util.OpenSearchUtil; +import org.opensearchmetrics.util.S3Util; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +public class GithubEventsLambda implements RequestHandler, Void> { + private final GithubEvents[] eventsToIndex = GithubEvents.getAllGithubEvents(); + private static final ServiceComponent COMPONENT = DaggerServiceComponent.create(); + private final OpenSearchUtil openSearchUtil; + private final S3Util s3Util; + private final ObjectMapper mapper; + + public GithubEventsLambda() { + this(COMPONENT.getOpenSearchUtil(), COMPONENT.getS3Util(), COMPONENT.getObjectMapper()); + } + + @VisibleForTesting + GithubEventsLambda(@NonNull OpenSearchUtil openSearchUtil, @NonNull S3Util s3Util, @NonNull ObjectMapper mapper) { + this.openSearchUtil = openSearchUtil; + this.s3Util = s3Util; + this.mapper = mapper; + } + + @Override + public Void handleRequest(Map input, Context context) { + LocalDate collectionStartDate; // UTC in the format yyyy-MM-dd + + // Reads Step Function Execution input in the format: + // { + // "collectionStartDate": "yyyy-MM-dd" + // } + // + // If not provided, defaults to yesterday + + if (input.containsKey("collectionStartDate")) { // user manually specified collection start date + collectionStartDate = LocalDate.parse(input.get("collectionStartDate")); + } else { // defaults to yesterday + collectionStartDate = LocalDate.now(ZoneOffset.UTC).minus(1, ChronoUnit.DAYS); + } + LocalDate collectionCurrentDate = collectionStartDate; + LocalDate today = LocalDate.now(ZoneOffset.UTC); + while (collectionCurrentDate.isBefore(today)) { + Map finalEventData = new HashMap<>(); + for (GithubEvents eventToIndex : eventsToIndex) { + String prefix = eventToIndex.getEventName() + "/" + collectionCurrentDate + "/"; + List objectKeys = s3Util.listObjectsKeys(prefix); + for (String objectKey : objectKeys) { + try (ResponseInputStream eventInputStream = s3Util.getObjectInputStream(objectKey)) { + JsonNode eventNode = mapper.readTree(eventInputStream); + EventData event = new EventData(); + event.setId(eventNode.path("id").textValue()); + event.setType(eventNode.path("name").textValue()); + event.setRepository(eventNode.path("payload").path("repository").path("name").textValue()); + event.setOrganization(eventNode.path("payload").path("organization").path("login").textValue()); + if (event.getOrganization() == null) { + event.setOrganization(eventNode.path("payload").path("repository").path("owner").path("login").textValue()); + } + event.setAction(eventNode.path("payload").path("action").textValue()); + event.setSender(eventNode.path("payload").path("sender").path("login").textValue()); + event.setCreatedAt(eventNode.path("uploaded_at").textValue()); + + finalEventData.put(event.getId(), event.getJson(event, mapper)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + String indexName = "github-user-activity-events-" + collectionCurrentDate.format(DateTimeFormatter.ofPattern("MM-yyyy")); + openSearchUtil.createIndexIfNotExists(indexName); + openSearchUtil.bulkIndex(indexName, finalEventData); + collectionCurrentDate = collectionCurrentDate.plusDays(1); + } + return null; + } +} diff --git a/src/main/java/org/opensearchmetrics/metrics/events/GithubEvents.java b/src/main/java/org/opensearchmetrics/metrics/events/GithubEvents.java new file mode 100644 index 0000000..02d919e --- /dev/null +++ b/src/main/java/org/opensearchmetrics/metrics/events/GithubEvents.java @@ -0,0 +1,32 @@ +package org.opensearchmetrics.metrics.events; + +import lombok.Getter; + +@Getter +public enum GithubEvents { + ISSUES_OPENED("issues.opened"), + ISSUES_CLOSED("issues.closed"), + ISSUES_LABELED("issues.labeled"), + ISSUES_UNLABELED("issues.unlabeled"), + ISSUES_TRANSFERRED("issues.transferred"), + ISSUES_ASSIGNED("issues.assigned"), + ISSUE_COMMENT_CREATED("issue_comment.created"), + PULL_REQUEST_CLOSED("pull_request.closed"), + PULL_REQUEST_OPENED("pull_request.opened"), + PULL_REQUEST_LABELED("pull_request.labeled"), + PULL_REQUEST_UNLABELED("pull_request.unlabeled"), + PULL_REQUEST_ASSIGNED("pull_request.assigned"), + PULL_REQUEST_REVIEW_SUBMITTED("pull_request_review.submitted"), + PULL_REQUEST_REVIEW_COMMENT_CREATED("pull_request_review_comment.created"), + GOLLUM("gollum"); + + private final String eventName; + + GithubEvents(String eventName) { + this.eventName = eventName; + } + + public static GithubEvents[] getAllGithubEvents() { + return values(); + } +} diff --git a/src/main/java/org/opensearchmetrics/metrics/release/ReleaseInputs.java b/src/main/java/org/opensearchmetrics/metrics/release/ReleaseInputs.java index eb5cc13..0d29163 100644 --- a/src/main/java/org/opensearchmetrics/metrics/release/ReleaseInputs.java +++ b/src/main/java/org/opensearchmetrics/metrics/release/ReleaseInputs.java @@ -8,7 +8,7 @@ public enum ReleaseInputs { VERSION_2_15_0("2.15.0", "closed", "2.15", true), VERSION_2_16_0("2.16.0", "closed", "2.16", true), VERSION_2_17_0("2.17.0", "closed", "2.17", true), - VERSION_2_18_0("2.18.0", "open", "2.x", true), + VERSION_2_18_0("2.18.0", "open", "2.18", true), VERSION_1_3_15("1.3.15", "closed", "1.3", false), VERSION_1_3_16("1.3.16", "closed", "1.3", false), VERSION_1_3_17("1.3.17", "closed", "1.3", false), diff --git a/src/main/java/org/opensearchmetrics/model/event/EventData.java b/src/main/java/org/opensearchmetrics/model/event/EventData.java new file mode 100644 index 0000000..79fc424 --- /dev/null +++ b/src/main/java/org/opensearchmetrics/model/event/EventData.java @@ -0,0 +1,55 @@ +package org.opensearchmetrics.model.event; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class EventData { + + @JsonProperty("id") + private String id; + + @JsonProperty("organization") + private String organization; + + @JsonProperty("repository") + private String repository; + + @JsonProperty("type") + private String type; + + @JsonProperty("action") + private String action; + + @JsonProperty("sender") + private String sender; + + @JsonProperty("created_at") + private String createdAt; + + public String toJson(ObjectMapper mapper) throws JsonProcessingException { + Map data = new HashMap<>(); + data.put("id", id); + data.put("organization", organization); + data.put("repository", repository); + data.put("type", type); + data.put("action", action); + data.put("sender", sender); + data.put("created_at", createdAt); + return mapper.writeValueAsString(data); + } + + public String getJson(EventData eventData, ObjectMapper objectMapper) { + try { + return eventData.toJson(objectMapper); + } catch (JsonProcessingException e) { + System.out.println("Error while serializing ReportDataRow to JSON " + e); + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/org/opensearchmetrics/util/S3Util.java b/src/main/java/org/opensearchmetrics/util/S3Util.java new file mode 100644 index 0000000..4ae7ec6 --- /dev/null +++ b/src/main/java/org/opensearchmetrics/util/S3Util.java @@ -0,0 +1,56 @@ +package org.opensearchmetrics.util; + +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class S3Util { + private final S3Client s3Client; + private final String bucketName; + + public S3Util(S3Client s3Client, String bucketName) { + this.s3Client = s3Client; + this.bucketName = bucketName; + } + + public ResponseInputStream getObjectInputStream(String objectName) { + try { + GetObjectRequest objectRequest = GetObjectRequest + .builder() + .key(objectName) + .bucket(bucketName) + .build(); + + return s3Client.getObject(objectRequest); + } catch (S3Exception e) { + System.err.println(e.awsErrorDetails().errorMessage()); + throw new RuntimeException("Failed to get object from S3", e); + } + } + + public List listObjectsKeys(String prefix) { + try { + ListObjectsV2Request listReq = ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(prefix) + .build(); + + ListObjectsV2Iterable listRes = s3Client.listObjectsV2Paginator(listReq); + + return listRes.stream() + .flatMap(r -> r.contents().stream()) + .map(S3Object::key) + .collect(Collectors.toCollection(ArrayList::new)); + } catch (S3Exception e) { + System.out.println(e.awsErrorDetails().errorMessage()); + throw new RuntimeException("Failed to list object keys from S3", e); + } + } +} diff --git a/src/test/java/org/opensearchmetrics/lambda/GithubEventsLambdaTest.java b/src/test/java/org/opensearchmetrics/lambda/GithubEventsLambdaTest.java new file mode 100644 index 0000000..f99f5ca --- /dev/null +++ b/src/test/java/org/opensearchmetrics/lambda/GithubEventsLambdaTest.java @@ -0,0 +1,158 @@ +package org.opensearchmetrics.lambda; + +import com.amazonaws.services.lambda.runtime.Context; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearchmetrics.util.OpenSearchUtil; +import org.opensearchmetrics.util.S3Util; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +public class GithubEventsLambdaTest { + @Mock + private ObjectMapper objectMapper; + + @Mock + private OpenSearchUtil openSearchUtil; + + @Mock + private S3Util s3Util; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + } + + @Test + public void testHandleRequestYesterday() { + // Arrange + ObjectMapper realMapper = new ObjectMapper(); + GithubEventsLambda githubEventsLambda = new GithubEventsLambda(openSearchUtil, s3Util, realMapper); + String eventJson = "{\"id\":\"123\",\"name\":\"push\",\"payload\":{\"repository\":{\"name\":\"myrepo\"},\"organization\":{\"login\":\"myorg\"},\"action\":\"created\",\"sender\":{\"login\":\"user\"}},\"uploaded_at\":\"2023-05-01T12:00:00Z\"}"; + List objectKeys = List.of("test_s3_key"); + GetObjectResponse getObjectResponse = mock(GetObjectResponse.class); + + when(s3Util.listObjectsKeys(anyString())).thenReturn(objectKeys); + when(s3Util.getObjectInputStream(anyString())).thenReturn(new ResponseInputStream<>(getObjectResponse, new ByteArrayInputStream(eventJson.getBytes()))); + + Map input = new HashMap<>(); + LocalDate yesterday = LocalDate.now(ZoneOffset.UTC).minus(1, ChronoUnit.DAYS); + input.put("collectionStartDate", yesterday.toString()); + + // Act + githubEventsLambda.handleRequest(input, mock(Context.class)); + + // Assert + String indexName = "github-user-activity-events-" + yesterday.format(DateTimeFormatter.ofPattern("MM-yyyy")); + verify(openSearchUtil).createIndexIfNotExists(indexName); + verify(openSearchUtil).bulkIndex(eq(indexName), any(Map.class)); + } + + @Test + public void testHandleRequestMonthAgo() { + // Arrange + ObjectMapper realMapper = new ObjectMapper(); + GithubEventsLambda githubEventsLambda = new GithubEventsLambda(openSearchUtil, s3Util, realMapper); + String eventJson = "{\"id\":\"123\",\"name\":\"push\",\"payload\":{\"repository\":{\"name\":\"myrepo\"},\"organization\":{\"login\":\"myorg\"},\"action\":\"created\",\"sender\":{\"login\":\"user\"}},\"uploaded_at\":\"2023-05-01T12:00:00Z\"}"; + List objectKeys = List.of("test_s3_key"); + GetObjectResponse getObjectResponse = mock(GetObjectResponse.class); + + when(s3Util.listObjectsKeys(anyString())).thenReturn(objectKeys); + when(s3Util.getObjectInputStream(anyString())).thenReturn(new ResponseInputStream<>(getObjectResponse, new ByteArrayInputStream(eventJson.getBytes()))); + + Map input = new HashMap<>(); + LocalDate today = LocalDate.now(ZoneOffset.UTC); + LocalDate lastMonth = today.minus(1, ChronoUnit.MONTHS); + input.put("collectionStartDate", lastMonth.toString()); + + // Act + githubEventsLambda.handleRequest(input, mock(Context.class)); + + // Assert + String indexNameLastMonth = "github-user-activity-events-" + lastMonth.format(DateTimeFormatter.ofPattern("MM-yyyy")); + verify(openSearchUtil, atLeastOnce()).createIndexIfNotExists(indexNameLastMonth); + verify(openSearchUtil, atLeastOnce()).bulkIndex(eq(indexNameLastMonth), any(Map.class)); + + String indexNameThisMonth = "github-user-activity-events-" + today.format(DateTimeFormatter.ofPattern("MM-yyyy")); + verify(openSearchUtil, atLeastOnce()).createIndexIfNotExists(indexNameThisMonth); + verify(openSearchUtil, atLeastOnce()).bulkIndex(eq(indexNameThisMonth), any(Map.class)); + } + + @Test + public void testHandleRequestDefault() { + // Arrange + ObjectMapper realMapper = new ObjectMapper(); + GithubEventsLambda githubEventsLambda = new GithubEventsLambda(openSearchUtil, s3Util, realMapper); + String eventJson = "{\"id\":\"123\",\"name\":\"push\",\"payload\":{\"repository\":{\"name\":\"myrepo\"},\"organization\":{\"login\":\"myorg\"},\"action\":\"created\",\"sender\":{\"login\":\"user\"}},\"uploaded_at\":\"2023-05-01T12:00:00Z\"}"; + List objectKeys = List.of("test_s3_key"); + GetObjectResponse getObjectResponse = mock(GetObjectResponse.class); + + when(s3Util.listObjectsKeys(anyString())).thenReturn(objectKeys); + when(s3Util.getObjectInputStream(anyString())).thenReturn(new ResponseInputStream<>(getObjectResponse, new ByteArrayInputStream(eventJson.getBytes()))); + + Map input = new HashMap<>(); + LocalDate yesterday = LocalDate.now(ZoneOffset.UTC).minus(1, ChronoUnit.DAYS); + + // Act + githubEventsLambda.handleRequest(input, mock(Context.class)); + + // Assert + String indexName = "github-user-activity-events-" + yesterday.format(DateTimeFormatter.ofPattern("MM-yyyy")); + verify(openSearchUtil).createIndexIfNotExists(indexName); + verify(openSearchUtil).bulkIndex(eq(indexName), any(Map.class)); + } + + @Test + public void testHandleRequestException() throws IOException { + // Arrange + GithubEventsLambda githubEventsLambda = new GithubEventsLambda(openSearchUtil, s3Util, objectMapper); + String eventJson = "{\"id\":\"123\",\"name\":\"push\",\"payload\":\"repository\":{\"name\":\"myrepo\"},\"organization\":{\"login\":\"myorg\"},\"action\":\"created\",\"sender\":{\"login\":\"user\"}},\"uploaded_at\":\"2023-05-01T12:00:00Z\"}"; + List objectKeys = List.of("test_s3_key"); + GetObjectResponse getObjectResponse = mock(GetObjectResponse.class); + + when(s3Util.listObjectsKeys(anyString())).thenReturn(objectKeys); + when(s3Util.getObjectInputStream(anyString())).thenReturn(new ResponseInputStream<>(getObjectResponse, new ByteArrayInputStream(eventJson.getBytes()))); + doThrow(new RuntimeException("Error running Github Events Lambda")).when(objectMapper).readTree(any(InputStream.class)); + + Map input = new HashMap<>(); + LocalDate yesterday = LocalDate.now(ZoneOffset.UTC).minus(1, ChronoUnit.DAYS); + input.put("collectionStartDate", yesterday.toString()); + + // Act & Assert + RuntimeException exception = assertThrows(RuntimeException.class, () -> + githubEventsLambda.handleRequest(input, mock(Context.class)) + ); + assertInstanceOf(RuntimeException.class, exception.getCause()); + } + + @Test + public void testHandleRequestDateException() { + Map input = new HashMap<>(); + input.put("collectionStartDate", "l;ajsd;fljk"); + + // Arrange + GithubEventsLambda githubEventsLambda = new GithubEventsLambda(openSearchUtil, s3Util, objectMapper); + assertThrows(DateTimeParseException.class, () -> + githubEventsLambda.handleRequest(input, mock(Context.class)) + ); + } +} diff --git a/src/test/java/org/opensearchmetrics/metrics/events/GithubEventsTest.java b/src/test/java/org/opensearchmetrics/metrics/events/GithubEventsTest.java new file mode 100644 index 0000000..17062f6 --- /dev/null +++ b/src/test/java/org/opensearchmetrics/metrics/events/GithubEventsTest.java @@ -0,0 +1,47 @@ +package org.opensearchmetrics.metrics.events; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class GithubEventsTest { + @Test + public void testGetEventName() { + assertEquals("issues.opened", GithubEvents.ISSUES_OPENED.getEventName()); + assertEquals("issues.closed", GithubEvents.ISSUES_CLOSED.getEventName()); + assertEquals("issues.labeled", GithubEvents.ISSUES_LABELED.getEventName()); + assertEquals("issues.unlabeled", GithubEvents.ISSUES_UNLABELED.getEventName()); + assertEquals("issues.transferred", GithubEvents.ISSUES_TRANSFERRED.getEventName()); + assertEquals("issues.assigned", GithubEvents.ISSUES_ASSIGNED.getEventName()); + assertEquals("issue_comment.created", GithubEvents.ISSUE_COMMENT_CREATED.getEventName()); + assertEquals("pull_request.closed", GithubEvents.PULL_REQUEST_CLOSED.getEventName()); + assertEquals("pull_request.opened", GithubEvents.PULL_REQUEST_OPENED.getEventName()); + assertEquals("pull_request.labeled", GithubEvents.PULL_REQUEST_LABELED.getEventName()); + assertEquals("pull_request.unlabeled", GithubEvents.PULL_REQUEST_UNLABELED.getEventName()); + assertEquals("pull_request.assigned", GithubEvents.PULL_REQUEST_ASSIGNED.getEventName()); + assertEquals("pull_request_review.submitted", GithubEvents.PULL_REQUEST_REVIEW_SUBMITTED.getEventName()); + assertEquals("pull_request_review_comment.created", GithubEvents.PULL_REQUEST_REVIEW_COMMENT_CREATED.getEventName()); + assertEquals("gollum", GithubEvents.GOLLUM.getEventName()); + } + + @Test + public void testGetAllGithubEvents() { + GithubEvents[] githubEvents = GithubEvents.getAllGithubEvents(); + assertEquals(15, githubEvents.length); + assertEquals(GithubEvents.ISSUES_OPENED, githubEvents[0]); + assertEquals(GithubEvents.ISSUES_CLOSED, githubEvents[1]); + assertEquals(GithubEvents.ISSUES_LABELED, githubEvents[2]); + assertEquals(GithubEvents.ISSUES_UNLABELED, githubEvents[3]); + assertEquals(GithubEvents.ISSUES_TRANSFERRED, githubEvents[4]); + assertEquals(GithubEvents.ISSUES_ASSIGNED, githubEvents[5]); + assertEquals(GithubEvents.ISSUE_COMMENT_CREATED, githubEvents[6]); + assertEquals(GithubEvents.PULL_REQUEST_CLOSED, githubEvents[7]); + assertEquals(GithubEvents.PULL_REQUEST_OPENED, githubEvents[8]); + assertEquals(GithubEvents.PULL_REQUEST_LABELED, githubEvents[9]); + assertEquals(GithubEvents.PULL_REQUEST_UNLABELED, githubEvents[10]); + assertEquals(GithubEvents.PULL_REQUEST_ASSIGNED, githubEvents[11]); + assertEquals(GithubEvents.PULL_REQUEST_REVIEW_SUBMITTED, githubEvents[12]); + assertEquals(GithubEvents.PULL_REQUEST_REVIEW_COMMENT_CREATED, githubEvents[13]); + assertEquals(GithubEvents.GOLLUM, githubEvents[14]); + } +} diff --git a/src/test/java/org/opensearchmetrics/metrics/release/ReleaseInputsTest.java b/src/test/java/org/opensearchmetrics/metrics/release/ReleaseInputsTest.java index 8ba2215..3ca478e 100644 --- a/src/test/java/org/opensearchmetrics/metrics/release/ReleaseInputsTest.java +++ b/src/test/java/org/opensearchmetrics/metrics/release/ReleaseInputsTest.java @@ -50,7 +50,7 @@ public void testGetBranch() { assertEquals("2.15", ReleaseInputs.VERSION_2_15_0.getBranch()); assertEquals("2.16", ReleaseInputs.VERSION_2_16_0.getBranch()); assertEquals("2.17", ReleaseInputs.VERSION_2_17_0.getBranch()); - assertEquals("2.x", ReleaseInputs.VERSION_2_18_0.getBranch()); + assertEquals("2.18", ReleaseInputs.VERSION_2_18_0.getBranch()); assertEquals("1.3", ReleaseInputs.VERSION_1_3_15.getBranch()); assertEquals("1.3", ReleaseInputs.VERSION_1_3_16.getBranch()); assertEquals("1.3", ReleaseInputs.VERSION_1_3_17.getBranch()); diff --git a/src/test/java/org/opensearchmetrics/model/event/EventDataTest.java b/src/test/java/org/opensearchmetrics/model/event/EventDataTest.java new file mode 100644 index 0000000..8ce3af1 --- /dev/null +++ b/src/test/java/org/opensearchmetrics/model/event/EventDataTest.java @@ -0,0 +1,140 @@ +package org.opensearchmetrics.model.event; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.when; + +public class EventDataTest { + @Mock + ObjectMapper objectMapper; + + private EventData eventData; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + eventData = new EventData(); + } + + @Test + public void testId() { + eventData.setId("123"); + assertEquals("123", eventData.getId()); + } + + @Test + public void testOrganization() { + eventData.setOrganization("exampleOrg"); + assertEquals("exampleOrg", eventData.getOrganization()); + } + + @Test + public void testRepository() { + eventData.setRepository("exampleRepo"); + assertEquals("exampleRepo", eventData.getRepository()); + } + + @Test + public void testType() { + eventData.setType("issues"); + assertEquals("issues", eventData.getType()); + } + + @Test + public void testAction() { + eventData.setAction("opened"); + assertEquals("opened", eventData.getAction()); + } + + @Test + public void testSender() { + eventData.setSender("Alejandro Rosalez"); + assertEquals("Alejandro Rosalez", eventData.getSender()); + } + + @Test + public void testCreatedAt() { + eventData.setCreatedAt("2024-09-23T20:14:08.346Z"); + assertEquals("2024-09-23T20:14:08.346Z", eventData.getCreatedAt()); + } + + @Test + void toJson() throws JsonProcessingException { + // Arrange + EventData eventData = new EventData(); + eventData.setId("1"); + eventData.setOrganization("test-org"); + eventData.setRepository("test-repo"); + eventData.setType("pull_request"); + eventData.setAction("closed"); + eventData.setSender("Alejandro Rosalez"); + eventData.setCreatedAt("2024-09-23T20:14:08.346Z"); + + Map expectedData = new HashMap<>(); + expectedData.put("id", "1"); + expectedData.put("organization", "test-org"); + expectedData.put("repository", "test-repo"); + expectedData.put("type", "pull_request"); + expectedData.put("action", "closed"); + expectedData.put("sender", "Alejandro Rosalez"); + expectedData.put("created_at", "2024-09-23T20:14:08.346Z"); + + when(objectMapper.writeValueAsString(expectedData)).thenReturn("expectedJson"); + + // Act + String actualJson = eventData.toJson(objectMapper); + + // Assert + assertEquals("expectedJson", actualJson); + } + + @Test + void getJson() throws JsonProcessingException { + // Arrange + EventData eventData = new EventData(); + eventData.setId("1"); + eventData.setOrganization("test-org"); + eventData.setRepository("test-repo"); + eventData.setType("pull_request"); + eventData.setAction("closed"); + eventData.setSender("Alejandro Rosalez"); + eventData.setCreatedAt("2024-09-23T20:14:08.346Z"); + + when(objectMapper.writeValueAsString(anyMap())).thenReturn("expectedJson"); + + // Act + String actualJson = eventData.getJson(eventData, objectMapper); + + // Assert + assertEquals("expectedJson", actualJson); + } + + @Test + void getJson_WithJsonProcessingException() throws JsonProcessingException { + // Arrange + EventData eventData = new EventData(); + eventData.setId("1"); + eventData.setOrganization("test-org"); + eventData.setRepository("test-repo"); + eventData.setType("pull_request"); + eventData.setAction("closed"); + eventData.setSender("Alejandro Rosalez"); + eventData.setCreatedAt("2024-09-23T20:14:08.346Z"); + + when(objectMapper.writeValueAsString(anyMap())).thenThrow(JsonProcessingException.class); + + // Act and Assert + assertThrows(RuntimeException.class, () -> eventData.getJson(eventData, objectMapper)); + } +} diff --git a/src/test/java/org/opensearchmetrics/util/S3UtilTest.java b/src/test/java/org/opensearchmetrics/util/S3UtilTest.java new file mode 100644 index 0000000..863e4d4 --- /dev/null +++ b/src/test/java/org/opensearchmetrics/util/S3UtilTest.java @@ -0,0 +1,105 @@ +package org.opensearchmetrics.util; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +public class S3UtilTest { + @Mock + private S3Client mockS3Client; + + private S3Util s3Util; + + @BeforeEach + void setup() { + MockitoAnnotations.openMocks(this); + s3Util = new S3Util(mockS3Client, "test_bucket_name"); + } + + @Test + public void WHEN_getObjectInputStream_THEN_return_ResponseInputStream() { + // Arrange + ResponseInputStream mockInputStream = mock(ResponseInputStream.class); + when(mockS3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockInputStream); + + // Act + ResponseInputStream inputStream = s3Util.getObjectInputStream("test_object_key"); + + // Assert + assertNotNull(inputStream); + assertEquals(inputStream, mockInputStream); + verify(mockS3Client).getObject(any(GetObjectRequest.class)); + } + + @Test + public void WHEN_getObjectInputStreamS3Exception_THEN_throw_Exception() { + // Arrange + S3Exception mockException = mock(S3Exception.class); + AwsErrorDetails mockAwsErrorDetails = mock(AwsErrorDetails.class); + when(mockS3Client.getObject(any(GetObjectRequest.class))).thenThrow(mockException); + when(mockException.awsErrorDetails()).thenReturn(mockAwsErrorDetails); + when(mockAwsErrorDetails.errorMessage()).thenReturn("Test Error Message"); + + // Act + RuntimeException exception = assertThrows(RuntimeException.class, () -> { + s3Util.getObjectInputStream("test_object_key"); + }); + + // Assert + assertEquals("Failed to get object from S3", exception.getMessage()); + assertInstanceOf(S3Exception.class, exception.getCause()); + } + + @Test + public void WHEN_listObjectKeys_THEN_return_ListObjectKeys() { + // Arrange + ListObjectsV2Response page = ListObjectsV2Response.builder() + .contents( + S3Object.builder().key("testPrefix/file1.txt").build(), + S3Object.builder().key("testPrefix/file2.txt").build() + ) + .build(); + ListObjectsV2Iterable listObjectsV2Iterable = mock(ListObjectsV2Iterable.class); + when(listObjectsV2Iterable.stream()).thenReturn(Stream.of(page)); + when(mockS3Client.listObjectsV2Paginator(any(ListObjectsV2Request.class))) + .thenReturn(listObjectsV2Iterable); + + // Act + List result = s3Util.listObjectsKeys("test_prefix"); + + // Assert + assertEquals(2, result.size()); + assertTrue(result.contains("testPrefix/file1.txt")); + assertTrue(result.contains("testPrefix/file2.txt")); + } + + @Test + public void WHEN_listObjectKeysS3Exception_THEN_throw_Exception() { + // Arrange + S3Exception mockException = mock(S3Exception.class); + AwsErrorDetails mockAwsErrorDetails = mock(AwsErrorDetails.class); + when(mockS3Client.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenThrow(mockException); + when(mockException.awsErrorDetails()).thenReturn(mockAwsErrorDetails); + when(mockAwsErrorDetails.errorMessage()).thenReturn("Test Error Message"); + + // Act & Assert + RuntimeException exception = assertThrows(RuntimeException.class, () -> + s3Util.listObjectsKeys("test_prefix") + ); + assertEquals("Failed to list object keys from S3", exception.getMessage()); + assertInstanceOf(S3Exception.class, exception.getCause()); + } + +}