[IT-3523] Upload test artifacts for branches beginning with test/

.github/workflows/deploy.yaml

@@ -20,7 +20,7 @@ jobs:
   deploy:
     name: Upload to Amazon S3
     runs-on: ubuntu-latest
-    if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }}
+    if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/test/') || startsWith(github.ref, 'refs/tags/v') }}
     needs: [ "validate" ]
     permissions:
       id-token: write

.pre-commit-config.yaml

@@ -1,3 +1,6 @@
+ci:
+  autoupdate_schedule: monthly
+
 repos:
 -   repo: https://github.com/awslabs/git-secrets
     rev: b9e96b3212fa06aea65964ff0d5cda84ce935f38
@@ -6,7 +9,7 @@ repos:
       entry: git-secrets
       args: [--scan, --recursive]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
     # On Windows, git will convert all CRLF to LF,
     # but only after all hooks are done executing.
@@ -18,20 +21,20 @@ repos:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/adrienverge/yamllint
-    rev: v1.32.0
+    rev: v1.35.1
     hooks:
     -   id: yamllint
 -   repo: https://github.com/aws-cloudformation/cfn-python-lint
-    rev: v0.83.1
+    rev: v1.20.2
     hooks:
     -   id: cfn-python-lint
         files: .*/.*\.(json|yml|yaml)$
 -   repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.4
+    rev: v1.5.5
     hooks:
     -   id: remove-tabs
 -   repo: https://github.com/sirosen/check-jsonschema
-    rev: 0.27.1
+    rev: 0.30.0
     hooks:
       - id: check-github-workflows
       - id: check-github-actions

README.md

@@ -12,27 +12,34 @@ This guide will help you deploy and manage your AWS ServiceCatalog using Infrast
 ## Deployment
 Merges to master will copy these templates to an S3 bucket that is used by [organizations-infra/sceptre/scipool](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool) to deploy stacks.
 
-## Development
-Currently there is a manual process for development. The steps are:
-1. Deploy a test service catalog product template to an S3 bucket that has public access.
-2. Reference that template in a template that deploys a product. Examples can be found in the individual
-   [Sceptre templates in org-formation-infra repo](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool/config/develop)
-
-Example:
-```yaml
-template:
-  path: "sc-product-ec2-linux-docker.j2"
-stack_name: "my-sc-product-ec2-linux-docker"
-parameters:
-  ProductName: "EC2: Linux Docker"
-sceptre_user_data:
-  # force cloudformation to update stack by setting a random number to the latest product's description
-  ProvisioningArtifactParameters: |
-    - Description: 'Test EC2 docker'
-      Info:
-        LoadTemplateFromURL: 'https://<TEST BUCKET>.s3.amazonaws.com/templates/ec2/sc-ec2-linux-docker.yaml'
-      Name: 'v9.9.9'
-```
+## Develoopment
+
+### Pre-Commit
+As a pre-deployment step we syntatically validate our sceptre and
+cloudformation yaml files with [pre-commit](https://pre-commit.com).
+
+Please install pre-commit, once installed the file validations will
+automatically run on every commit. Alternatively you can manually
+execute the validations by running `pre-commit run --all-files`.
+Please install pre-commit, once installed the file validations will
+automatically run on every commit.
+
+### Functional Testing
+In order to test a change to a template, the template to test needs
+to be uploaded to S3, and then a service catalog product needs to
+be configured to use the test template.
+
+This process has been automated by running the deploy pipeline on
+branches that begin with `test/`, which will upload the templates
+to S3 in a directory named after the branch name. Automating the
+process increases consistency and accuracy of the tests, and
+reduces the impact of potential human error.
+
+Once a test template has been uploaded to S3, a PR can be made
+to `organizations-infra` to create or update a service catalog
+product that references the test template, like in [this PR](https://github.com/Sage-Bionetworks-IT/organizations-infra/pull/1109/files).
+This process is outlined [in scipool documentation in `organizations-infra`](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool/README.md)
+and existing templates can be found in the individual [sceptre templates in `organiationss-infra`](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool/config/develop).
 
 ### Account and Credentials
 While the templates in this library could be used in any AWS account, Sage Bionetworks is currently deploying to the "scipooldev" and "scipoolprod" account.

templates/batch/sc-batch-fargate.yaml

@@ -206,6 +206,7 @@ Resources:
         config:
           ignore_checks:  # Ignore cfn-lint errors for PyPlate.
             - E3002
+            - E3012
     Type: AWS::Batch::JobDefinition
     Properties:
       Type: container

templates/ec2/sc-ec2-linux-docker-notebook.yaml

@@ -5,6 +5,7 @@ Metadata:
     config:
       ignore_checks:
         - E7001
+        - W6001
   AWS::CloudFormation::Interface:
     ParameterGroups:
       - Label:
@@ -155,8 +156,6 @@ Resources:
         'Fn::FindInMap': [AccountToImportParams, !Ref "AWS::AccountId", VPCId]
       SecurityGroupIngress:
         - CidrIp: "10.50.0.0/16"
-          FromPort: -1
-          ToPort: -1
           IpProtocol: "-1"
 
   InstanceProfile:
@@ -274,7 +273,7 @@ Resources:
           files:
             /opt/sage/bin/make_env_vars_file.sh:
               source: "https://raw.githubusercontent.com/Sage-Bionetworks/service-catalog-utils/v2.0.3/linux/opt/sage/bin/make_env_vars_file.sh"
-              mode: "00744"
+              mode: "000744"
               owner: "root"
               group: "root"
           commands:

templates/ec2/sc-ec2-linux-docker.yaml

@@ -86,8 +86,6 @@ Resources:
         'Fn::FindInMap': [AccountToImportParams, !Ref "AWS::AccountId", VPCId]
       SecurityGroupIngress:
         - CidrIp: "10.50.0.0/16"
-          FromPort: -1
-          ToPort: -1
           IpProtocol: "-1"
   InstanceRole:
     Type: AWS::IAM::Role
@@ -176,7 +174,7 @@ Resources:
           files:
             /opt/sage/bin/make_env_vars_file.sh:
               source: "https://raw.githubusercontent.com/Sage-Bionetworks/service-catalog-utils/v2.1.0/linux/opt/sage/bin/make_env_vars_file.sh"
-              mode: "00744"
+              mode: "000744"
               owner: "root"
               group: "root"
           commands:

templates/ec2/sc-ec2-windows-jumpcloud.yaml

@@ -114,7 +114,7 @@ Resources:
           files:
             'c:\\scripts\\install-chocolatey.ps1':
               source: "https://chocolatey.org/install.ps1"
-              mode: "0664"
+              mode: "000664"
           commands:
             01_install_nuget:
               command: 'Powershell.exe Install-PackageProvider -Name NuGet -Force'
@@ -133,7 +133,7 @@ Resources:
           files:
             'c:\\scripts\\set_env_vars_file.ps1':
               source: "https://raw.githubusercontent.com/Sage-Bionetworks/infra-utils/v1.0.9/aws/set_env_vars_file.ps1"
-              mode: "0664"
+              mode: "000664"
           commands:
             01_set_env_vars:
               command: !Join
@@ -145,10 +145,10 @@ Resources:
           files:
             'c:\scripts\install-ms-vc.ps1':
               source: "https://raw.githubusercontent.com/Sage-Bionetworks/infra-utils/v1.0.8/aws/install-ms-vc.ps1"
-              mode: "0664"
+              mode: "000664"
             'c:\\scripts\\install-jc-agent.ps1':
               source: "https://raw.githubusercontent.com/TheJumpCloud/support/master/scripts/windows/InstallWindowsAgent.ps1"
-              mode: "0664"
+              mode: "000664"
           commands:
             01_install_ms_vc:
               command: !Join
@@ -220,8 +220,6 @@ Resources:
         'Fn::FindInMap': [AccountToImportParams, !Ref "AWS::AccountId", VPCId]
       SecurityGroupIngress:
         - CidrIp: "10.50.0.0/16"
-          FromPort: -1
-          ToPort: -1
           IpProtocol: "-1"
 Outputs:
   WindowsInstancePrivateIpAddress:

templates/s3/sc-s3-synapse-ra.yaml

@@ -84,6 +84,7 @@ Resources:
         config:
           ignore_checks:
             - E3001
+            - E3006
     Properties:
       Target:
         Bucket: !Ref S3Bucket