diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index f8720d4..49b026f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -25,6 +25,7 @@ So you want to contribute code to this project? Excellent! We're glad you're her - `cdk deploy OpenSearchMetrics-GitHubAutomationApp-Secret`: Creates the GitHub app secret which will be used during the GitHub app runtime. - `cdk deploy OpenSearchMetrics-GitHubWorkflowMonitor-Alarms`: Creates the Alarms to Monitor the Critical GitHub CI workflows by the GitHub Automation App. - `cdk deploy OpenSearchMetrics-GitHubAutomationApp`: Create the resources which launches the [GitHub Automation App](https://github.com/opensearch-project/automation-app). Listens to GitHub events and index the data to Metrics cluster. + - `cdk deploy OpenSearchMetrics-GitHubAutomationAppEvents-S3`: Creates the S3 Bucket for the [GitHub Automation App](https://github.com/opensearch-project/automation-app) to store OpenSearch Project GitHub Events. ### Forking and Cloning diff --git a/infrastructure/lib/infrastructure-stack.ts b/infrastructure/lib/infrastructure-stack.ts index 72fb9bd..7171ff3 100644 --- a/infrastructure/lib/infrastructure-stack.ts +++ b/infrastructure/lib/infrastructure-stack.ts @@ -20,6 +20,7 @@ import { OpenSearchMetricsNginxCognito } from "./constructs/opensearchNginxProxy import { OpenSearchMetricsMonitoringStack } from "./stacks/monitoringDashboard"; import { OpenSearchMetricsSecretsStack } from "./stacks/secrets"; import { GitHubAutomationApp } from "./stacks/gitHubAutomationApp"; +import { OpenSearchS3 } from "./stacks/s3"; import { GitHubWorkflowMonitorAlarms } from "./stacks/gitHubWorkflowMonitorAlarms"; export class InfrastructureStack extends Stack { @@ -47,6 +48,9 @@ export class InfrastructureStack extends Stack { ], }); + // Create S3 bucket for the GitHub Events + const openSearchEventsS3Bucket = new OpenSearchS3(app, "OpenSearchMetrics-GitHubAutomationAppEvents-S3"); + // Create resources to launch the GitHub Automation App const gitHubAutomationApp = new GitHubAutomationApp(app, "OpenSearchMetrics-GitHubAutomationApp", { vpc: vpcStack.vpc, @@ -54,8 +58,9 @@ export class InfrastructureStack extends Stack { account: Project.AWS_ACCOUNT, ami: Project.EC2_AMI_SSM.toString(), secret: openSearchMetricsGitHubAutomationAppSecretStack.secret, - workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn - }) + workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn, + githubEventsBucketArn: openSearchEventsS3Bucket.bucket.bucketArn + }); // Create OpenSearch Domain, roles, permissions, cognito setup, cross account OpenSearch access for jenkins diff --git a/infrastructure/lib/stacks/gitHubAutomationApp.ts b/infrastructure/lib/stacks/gitHubAutomationApp.ts index b8c9be9..e77307d 100644 --- a/infrastructure/lib/stacks/gitHubAutomationApp.ts +++ b/infrastructure/lib/stacks/gitHubAutomationApp.ts @@ -42,6 +42,7 @@ export interface GitHubAppProps { readonly ami?: string readonly secret: Secret; readonly workflowAlarmsArn: string[]; + readonly githubEventsBucketArn: string; } @@ -53,11 +54,11 @@ export class GitHubAutomationApp extends Stack { constructor(scope: Construct, id: string, props: GitHubAppProps) { super(scope, id); - const instanceRole = this.createInstanceRole(props.secret.secretArn, props.account, props.workflowAlarmsArn); + const instanceRole = this.createInstanceRole(props.secret.secretArn, props.account, props.workflowAlarmsArn, props.githubEventsBucketArn); this.githubAppRole = instanceRole; this.asg = new AutoScalingGroup(this, 'OpenSearchMetrics-GitHubAutomationApp-Asg', { - instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.LARGE), + instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE), blockDevices: [{deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(20)}], healthCheck: HealthCheck.ec2({grace: Duration.seconds(90)}), machineImage: props && props.ami ? @@ -92,7 +93,7 @@ export class GitHubAutomationApp extends Stack { this.asg.addUserData(...this.getUserData(props.secret.secretName)); } - private createInstanceRole(secretArn: string, account: string, alarmsArn: string[]): Role { + private createInstanceRole(secretArn: string, account: string, alarmsArn: string[], githubEventsBucketArn: string): Role { const role = new Role(this, "OpenSearchMetrics-GitHubAutomationApp-Role", { assumedBy: new CompositePrincipal( new ServicePrincipal('ec2.amazonaws.com'), @@ -137,6 +138,15 @@ export class GitHubAutomationApp extends Stack { }), ); + role.addToPolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "s3:PutObject", + ], + resources: [`${githubEventsBucketArn}/*`], + }), + ); return role; } @@ -154,7 +164,8 @@ export class GitHubAutomationApp extends Stack { `aws secretsmanager get-secret-value --secret-id ${secretName} --query SecretString --output text >> automation-app/.env`, 'cd automation-app/docker', 'PORT=8080 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-merged-pulls-monitor.yml docker-compose -p github-merged-pulls-monitor up -d', - 'PORT=8081 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-workflow-runs-monitor.yml docker-compose -p github-workflow-runs-monitor up -d' + 'PORT=8081 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-workflow-runs-monitor.yml docker-compose -p github-workflow-runs-monitor up -d', + 'PORT=8082 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-events-to-s3.yml docker-compose -p github-events-to-s3 up -d', ]; } } diff --git a/infrastructure/lib/stacks/s3.ts b/infrastructure/lib/stacks/s3.ts new file mode 100644 index 0000000..cfae047 --- /dev/null +++ b/infrastructure/lib/stacks/s3.ts @@ -0,0 +1,29 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import {Stack, RemovalPolicy, StackProps} from "aws-cdk-lib"; +import { Bucket, BlockPublicAccess, BucketEncryption } from 'aws-cdk-lib/aws-s3'; +import { Construct } from "constructs"; + +export class OpenSearchS3 extends Stack { + + public readonly bucket: Bucket; + + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id); + + this.bucket = new Bucket(this, 'OpenSearchS3Bucket', { + publicReadAccess: false, + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + encryption: BucketEncryption.S3_MANAGED, + enforceSSL: true, + versioned: true, + removalPolicy: RemovalPolicy.RETAIN, + }); + } +} diff --git a/infrastructure/lib/stacks/vpc.ts b/infrastructure/lib/stacks/vpc.ts index cf7dc70..290413d 100644 --- a/infrastructure/lib/stacks/vpc.ts +++ b/infrastructure/lib/stacks/vpc.ts @@ -11,7 +11,6 @@ import { Stack, StackProps } from "aws-cdk-lib"; import { Peer, Port, SecurityGroup, SelectedSubnets, SubnetType, Vpc } from 'aws-cdk-lib/aws-ec2'; import { Construct } from "constructs"; - export class VpcStack extends Stack { public readonly vpc: Vpc; public readonly securityGroup: SecurityGroup diff --git a/infrastructure/test/gitHubAutomationApp-stack.test.ts b/infrastructure/test/gitHubAutomationApp-stack.test.ts index 77522f9..5c8c867 100644 --- a/infrastructure/test/gitHubAutomationApp-stack.test.ts +++ b/infrastructure/test/gitHubAutomationApp-stack.test.ts @@ -13,6 +13,7 @@ import { VpcStack } from "../lib/stacks/vpc"; import { GitHubAutomationApp } from "../lib/stacks/gitHubAutomationApp"; import { OpenSearchMetricsSecretsStack } from "../lib/stacks/secrets"; import {GitHubWorkflowMonitorAlarms} from "../lib/stacks/gitHubWorkflowMonitorAlarms"; +import {OpenSearchS3} from "../lib/stacks/s3"; test('OpenSearch GitHub App Stack test ', () => { @@ -21,6 +22,7 @@ test('OpenSearch GitHub App Stack test ', () => { const openSearchMetricsGitHubAppSecretStack = new OpenSearchMetricsSecretsStack(app, "Test-OpenSearchMetrics-GitHubAutomationApp-Secret", { secretName: 'test-github-app-creds' }); + const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3"); const gitHubWorkflowMonitorAlarms = new GitHubWorkflowMonitorAlarms(app, "Test-OpenSearchMetrics-GitHubWorkflowMonitor-Alarms", { namespace: 'GitHubActions', @@ -36,7 +38,8 @@ test('OpenSearch GitHub App Stack test ', () => { account: Project.AWS_ACCOUNT, ami: Project.EC2_AMI_SSM.toString(), secret: openSearchMetricsGitHubAppSecretStack.secret, - workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn + workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn, + githubEventsBucketArn: s3Stack.bucket.bucketArn }); const template = Template.fromStack(gitHubApp); @@ -90,4 +93,26 @@ test('OpenSearch GitHub App Stack test ', () => { ]) } }); + + template.hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: Match.arrayWith([ + Match.objectLike({ + Action: "s3:PutObject", + Effect: "Allow", + Resource: { + "Fn::Join": [ + "", + [ + { + "Fn::ImportValue": Match.stringLikeRegexp('Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3:ExportsOutputFnGetAttOpenSearchS3Bucket.*') + }, + "/*" + ] + ] + } + }) + ]) + } + }); }); diff --git a/infrastructure/test/s3-stack.test.ts b/infrastructure/test/s3-stack.test.ts new file mode 100644 index 0000000..91fb0e5 --- /dev/null +++ b/infrastructure/test/s3-stack.test.ts @@ -0,0 +1,84 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { App } from "aws-cdk-lib"; +import { Template } from "aws-cdk-lib/assertions"; +import {OpenSearchS3} from "../lib/stacks/s3"; + +test('S3 Stack Test', () => { + const app = new App(); + const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3"); + const s3StackTemplate = Template.fromStack(s3Stack); + s3StackTemplate.resourceCountIs('AWS::S3::Bucket', 1); + s3StackTemplate.resourceCountIs('AWS::S3::BucketPolicy', 1); + s3StackTemplate.hasResourceProperties('AWS::S3::Bucket', { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "ServerSideEncryptionByDefault": { + "SSEAlgorithm": "AES256" + } + } + ] + }, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "VersioningConfiguration": { + "Status": "Enabled" + } + }); + s3StackTemplate.hasResourceProperties('AWS::S3::BucketPolicy', { + "Bucket": { + "Ref": "OpenSearchS3Bucket2ED683CC" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "OpenSearchS3Bucket2ED683CC", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "OpenSearchS3Bucket2ED683CC", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + }); + +});