Added functionality to store github events in s3 bucket

DEVELOPER_GUIDE.md

@@ -25,6 +25,7 @@ So you want to contribute code to this project? Excellent! We're glad you're her
     - `cdk deploy OpenSearchMetrics-GitHubAutomationApp-Secret`: Creates the GitHub app secret which will be used during the GitHub app runtime.
     - `cdk deploy OpenSearchMetrics-GitHubWorkflowMonitor-Alarms`: Creates the Alarms to Monitor the Critical GitHub CI workflows by the GitHub Automation App.
     - `cdk deploy OpenSearchMetrics-GitHubAutomationApp`: Create the resources which launches the [GitHub Automation App](https://github.com/opensearch-project/automation-app). Listens to GitHub events and index the data to Metrics cluster.
+    - `cdk deploy OpenSearchMetrics-GitHubAutomationAppEvents-S3`: Creates the S3 Bucket for the [GitHub Automation App](https://github.com/opensearch-project/automation-app) to store OpenSearch Project GitHub Events.
 
 ### Forking and Cloning
 

infrastructure/lib/infrastructure-stack.ts

@@ -20,6 +20,7 @@ import { OpenSearchMetricsNginxCognito } from "./constructs/opensearchNginxProxy
 import { OpenSearchMetricsMonitoringStack } from "./stacks/monitoringDashboard";
 import { OpenSearchMetricsSecretsStack } from "./stacks/secrets";
 import { GitHubAutomationApp } from "./stacks/gitHubAutomationApp";
+import { OpenSearchS3 } from "./stacks/s3";
 import { GitHubWorkflowMonitorAlarms } from "./stacks/gitHubWorkflowMonitorAlarms";
 
 export class InfrastructureStack extends Stack {
@@ -47,15 +48,19 @@ export class InfrastructureStack extends Stack {
       ],
     });
 
+    // Create S3 bucket for the GitHub Events
+    const openSearchEventsS3Bucket = new OpenSearchS3(app, "OpenSearchMetrics-GitHubAutomationAppEvents-S3");
+
     // Create resources to launch the GitHub Automation App
     const gitHubAutomationApp = new GitHubAutomationApp(app, "OpenSearchMetrics-GitHubAutomationApp", {
       vpc: vpcStack.vpc,
       region: Project.REGION,
       account: Project.AWS_ACCOUNT,
       ami: Project.EC2_AMI_SSM.toString(),
       secret: openSearchMetricsGitHubAutomationAppSecretStack.secret,
-      workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn
-    })
+      workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn,
+      githubEventsBucketArn: openSearchEventsS3Bucket.bucket.bucketArn
+    });
 
 
     // Create OpenSearch Domain, roles, permissions, cognito setup, cross account OpenSearch access for jenkins

infrastructure/lib/stacks/gitHubAutomationApp.ts

@@ -42,6 +42,7 @@ export interface GitHubAppProps {
     readonly ami?: string
     readonly secret: Secret;
     readonly workflowAlarmsArn: string[];
+    readonly githubEventsBucketArn: string;
 }
 
 
@@ -53,11 +54,11 @@ export class GitHubAutomationApp extends Stack {
     constructor(scope: Construct, id: string, props: GitHubAppProps) {
         super(scope, id);
 
-        const instanceRole = this.createInstanceRole(props.secret.secretArn, props.account, props.workflowAlarmsArn);
+        const instanceRole = this.createInstanceRole(props.secret.secretArn, props.account, props.workflowAlarmsArn, props.githubEventsBucketArn);
         this.githubAppRole = instanceRole;
 
         this.asg = new AutoScalingGroup(this, 'OpenSearchMetrics-GitHubAutomationApp-Asg', {
-            instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.LARGE),
+            instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE),
             blockDevices: [{deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(20)}],
             healthCheck: HealthCheck.ec2({grace: Duration.seconds(90)}),
             machineImage: props && props.ami ?
@@ -92,7 +93,7 @@ export class GitHubAutomationApp extends Stack {
         this.asg.addUserData(...this.getUserData(props.secret.secretName));
     }
 
-    private createInstanceRole(secretArn: string, account: string, alarmsArn: string[]): Role {
+    private createInstanceRole(secretArn: string, account: string, alarmsArn: string[], githubEventsBucketArn: string): Role {
         const role = new Role(this, "OpenSearchMetrics-GitHubAutomationApp-Role", {
             assumedBy: new CompositePrincipal(
                 new ServicePrincipal('ec2.amazonaws.com'),
@@ -137,6 +138,15 @@ export class GitHubAutomationApp extends Stack {
 
             }),
         );
+        role.addToPolicy(
+            new PolicyStatement({
+                effect: Effect.ALLOW,
+                actions: [
+                    "s3:PutObject",
+                ],
+                resources: [`${githubEventsBucketArn}/*`],
+            }),
+        );
         return role;
     }
 
@@ -154,7 +164,8 @@ export class GitHubAutomationApp extends Stack {
             `aws secretsmanager get-secret-value --secret-id ${secretName} --query SecretString --output text >> automation-app/.env`,
             'cd automation-app/docker',
             'PORT=8080 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-merged-pulls-monitor.yml docker-compose -p github-merged-pulls-monitor up -d',
-            'PORT=8081 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-workflow-runs-monitor.yml docker-compose -p github-workflow-runs-monitor up -d'
+            'PORT=8081 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-workflow-runs-monitor.yml docker-compose -p github-workflow-runs-monitor up -d',
+            'PORT=8082 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-events-to-s3.yml docker-compose -p github-events-to-s3 up -d',
         ];
     }
 }

infrastructure/lib/stacks/s3.ts

@@ -0,0 +1,29 @@
+/**
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import {Stack, RemovalPolicy, StackProps} from "aws-cdk-lib";
+import { Bucket, BlockPublicAccess, BucketEncryption } from 'aws-cdk-lib/aws-s3';
+import { Construct } from "constructs";
+
+export class OpenSearchS3 extends Stack {
+
+    public readonly bucket: Bucket;
+
+    constructor(scope: Construct, id: string, props?: StackProps) {
+        super(scope, id);
+
+        this.bucket = new Bucket(this, 'OpenSearchS3Bucket', {
+            publicReadAccess: false,
+            blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
+            encryption: BucketEncryption.S3_MANAGED,
+            enforceSSL: true,
+            versioned: true,
+            removalPolicy: RemovalPolicy.RETAIN,
+        });
+    }
+}

infrastructure/lib/stacks/vpc.ts

@@ -11,7 +11,6 @@ import { Stack, StackProps } from "aws-cdk-lib";
 import { Peer, Port, SecurityGroup, SelectedSubnets, SubnetType, Vpc } from 'aws-cdk-lib/aws-ec2';
 import { Construct } from "constructs";
 
-
 export class VpcStack extends Stack {
     public readonly vpc: Vpc;
     public readonly securityGroup: SecurityGroup

infrastructure/test/gitHubAutomationApp-stack.test.ts

@@ -13,6 +13,7 @@ import { VpcStack } from "../lib/stacks/vpc";
 import { GitHubAutomationApp } from "../lib/stacks/gitHubAutomationApp";
 import { OpenSearchMetricsSecretsStack } from "../lib/stacks/secrets";
 import {GitHubWorkflowMonitorAlarms} from "../lib/stacks/gitHubWorkflowMonitorAlarms";
+import {OpenSearchS3} from "../lib/stacks/s3";
 
 
 test('OpenSearch GitHub App Stack test ', () => {
@@ -21,6 +22,7 @@ test('OpenSearch GitHub App Stack test ', () => {
     const openSearchMetricsGitHubAppSecretStack = new OpenSearchMetricsSecretsStack(app, "Test-OpenSearchMetrics-GitHubAutomationApp-Secret", {
         secretName: 'test-github-app-creds'
     });
+    const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3");
 
     const gitHubWorkflowMonitorAlarms = new GitHubWorkflowMonitorAlarms(app, "Test-OpenSearchMetrics-GitHubWorkflowMonitor-Alarms", {
         namespace: 'GitHubActions',
@@ -36,7 +38,8 @@ test('OpenSearch GitHub App Stack test ', () => {
         account: Project.AWS_ACCOUNT,
         ami: Project.EC2_AMI_SSM.toString(),
         secret: openSearchMetricsGitHubAppSecretStack.secret,
-        workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn
+        workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn,
+        githubEventsBucketArn: s3Stack.bucket.bucketArn
     });
 
     const template = Template.fromStack(gitHubApp);
@@ -90,4 +93,26 @@ test('OpenSearch GitHub App Stack test ', () => {
             ])
         }
     });
+
+    template.hasResourceProperties('AWS::IAM::Policy', {
+        PolicyDocument: {
+            Statement: Match.arrayWith([
+                Match.objectLike({
+                    Action: "s3:PutObject",
+                    Effect: "Allow",
+                    Resource: {
+                        "Fn::Join": [
+                            "",
+                            [
+                                {
+                                    "Fn::ImportValue": Match.stringLikeRegexp('Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3:ExportsOutputFnGetAttOpenSearchS3Bucket.*')
+                                },
+                                "/*"
+                            ]
+                        ]
+                    }
+                })
+            ])
+        }
+    });
 });

infrastructure/test/s3-stack.test.ts

@@ -0,0 +1,84 @@
+/**
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import { App } from "aws-cdk-lib";
+import { Template } from "aws-cdk-lib/assertions";
+import {OpenSearchS3} from "../lib/stacks/s3";
+
+test('S3 Stack Test', () => {
+    const app = new App();
+    const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3");
+    const s3StackTemplate = Template.fromStack(s3Stack);
+    s3StackTemplate.resourceCountIs('AWS::S3::Bucket', 1);
+    s3StackTemplate.resourceCountIs('AWS::S3::BucketPolicy', 1);
+    s3StackTemplate.hasResourceProperties('AWS::S3::Bucket', {
+        "BucketEncryption": {
+            "ServerSideEncryptionConfiguration": [
+                {
+                    "ServerSideEncryptionByDefault": {
+                        "SSEAlgorithm": "AES256"
+                    }
+                }
+            ]
+        },
+        "PublicAccessBlockConfiguration": {
+            "BlockPublicAcls": true,
+            "BlockPublicPolicy": true,
+            "IgnorePublicAcls": true,
+            "RestrictPublicBuckets": true
+        },
+        "VersioningConfiguration": {
+            "Status": "Enabled"
+        }
+    });
+    s3StackTemplate.hasResourceProperties('AWS::S3::BucketPolicy', {
+        "Bucket": {
+            "Ref": "OpenSearchS3Bucket2ED683CC"
+        },
+        "PolicyDocument": {
+            "Statement": [
+                {
+                    "Action": "s3:*",
+                    "Condition": {
+                        "Bool": {
+                            "aws:SecureTransport": "false"
+                        }
+                    },
+                    "Effect": "Deny",
+                    "Principal": {
+                        "AWS": "*"
+                    },
+                    "Resource": [
+                        {
+                            "Fn::GetAtt": [
+                                "OpenSearchS3Bucket2ED683CC",
+                                "Arn"
+                            ]
+                        },
+                        {
+                            "Fn::Join": [
+                                "",
+                                [
+                                    {
+                                        "Fn::GetAtt": [
+                                            "OpenSearchS3Bucket2ED683CC",
+                                            "Arn"
+                                        ]
+                                    },
+                                    "/*"
+                                ]
+                            ]
+                        }
+                    ]
+                }
+            ],
+            "Version": "2012-10-17"
+        }
+    });
+
+});