diff --git a/.github/workflows/deploy-prd-enhanced-worker.yml b/.github/workflows/deploy-prd-enhanced-worker.yml index 10eb08325..7f758994b 100644 --- a/.github/workflows/deploy-prd-enhanced-worker.yml +++ b/.github/workflows/deploy-prd-enhanced-worker.yml @@ -21,7 +21,7 @@ jobs: aws-region: us-east-2 - name: Deploy Enhanced Infrastructure run: | + npm ci cd cdk-infra/ npm ci npm run deploy:enhanced:worker -- -c env=prd -c customFeatureName=enhancedApp - diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index c779ff49b..4690585f6 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -25,6 +25,7 @@ jobs: aws-region: us-east-2 - name: Deploy Enhanced Infrastructure run: | + npm ci cd cdk-infra/ npm ci npm run deploy:feature:stack -- -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg auto-builder-stack-enhancedApp-dotcomstg-worker diff --git a/Dockerfile b/Dockerfile index bab432eca..4c6b50500 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ RUN cd ./modules/oas-page-builder \ # where repo work will happen FROM ubuntu:20.04 ARG WORK_DIRECTORY=/home/docsworker-xlarge -ARG SNOOTY_PARSER_VERSION=0.16.3 -ARG SNOOTY_FRONTEND_VERSION=0.16.6 +ARG SNOOTY_PARSER_VERSION=0.16.4 +ARG SNOOTY_FRONTEND_VERSION=0.16.8 ARG MUT_VERSION=0.11.1 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH @@ -63,17 +63,31 @@ USER docsworker-xlarge WORKDIR ${WORK_DIRECTORY} +# Get Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +RUN chmod -R 777 ${WORK_DIRECTORY}/.cargo/bin + +ENV PATH="${WORK_DIRECTORY}/.cargo/bin:${PATH}" + + # get shared.mk RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk # install snooty frontend and docs-tools RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ && cd snooty \ - && npm ci --legacy-peer-deps --omit=dev \ + # Need to remove omit dev as the filter functionality for the frontend depends on a dev dependency. + && npm ci --legacy-peer-deps \ && git clone --depth 1 https://github.com/mongodb/docs-tools.git \ && mkdir -p ./static/images \ && mv ./docs-tools/themes/mongodb/static ./static/docs-tools \ - && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg + && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg \ + && cd component-factory-transformer \ + && cargo build \ + && rustup target add wasm32-wasi \ + && npm run prepublishOnly + # install redoc fork RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \ diff --git a/Dockerfile.local b/Dockerfile.local index 4382caf4d..b918c4860 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,8 +1,8 @@ FROM arm64v8/ubuntu:20.04 as initial ARG NPM_BASE_64_AUTH ARG NPM_EMAIL -ARG SNOOTY_PARSER_VERSION=0.15.2 -ARG SNOOTY_FRONTEND_VERSION=0.15.7 +ARG SNOOTY_PARSER_VERSION=0.16.4 +ARG SNOOTY_FRONTEND_VERSION=0.16.8 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH @@ -24,6 +24,8 @@ RUN apt-get install --yes build-essential RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config + + RUN python3 -m pip install poetry # install snooty parser @@ -43,16 +45,32 @@ RUN git clone -b v${MUT_VERSION} --depth 1 https://github.com/mongodb/mut.git \ ENV PATH="${PATH}:/opt/snooty:/opt/mut:/${WORK_DIRECTORY}/.local/bin" + + # setup user and root directory RUN useradd -ms /bin/bash docsworker-xlarge RUN chmod 755 -R ${WORK_DIRECTORY} RUN chown -Rv docsworker-xlarge ${WORK_DIRECTORY} + USER docsworker-xlarge +# Get Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +RUN chmod -R 777 ${WORK_DIRECTORY}/.cargo/bin + +ENV PATH="${WORK_DIRECTORY}/.cargo/bin:${PATH}" +RUN echo ${WORK_DIRECTORY} + + # install snooty frontend and docs-tools RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ && cd snooty \ - && npm ci --legacy-peer-deps --omit=dev + && npm ci --legacy-peer-deps \ + && cd component-factory-transformer \ + && cargo build \ + && rustup target add wasm32-wasi \ + && npm run prepublishOnly RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk @@ -61,7 +79,7 @@ RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.co # Install dependencies for Redoc CLI && cd redoc/ \ && npm ci --prefix cli/ --omit=dev - + FROM initial as persistence RUN mkdir -p modules/persistence && chmod 755 modules/persistence diff --git a/README.md b/README.md index 7936fe161..4076e0470 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Docs Worker Pool -As part of the Docs Tools Next Generation Project, the Docs Worker Pool seeks to make the build process for developers +As part of the Docs Tools Next Generation Project, the Docs Worker Pool seeks to make the build process for developers both easier and more scalable for developers. The Docs Worker Pool operates on ECS Fargate. The serverless framework and cloudformation are used to automate diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index eed59edfb..bf6548933 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -401,6 +401,12 @@ export abstract class JobHandler { envVars += `PATH_PREFIX=${pathPrefix}\n`; } const snootyFrontEndVars = { + // Setting this to always be true. It's one less SSM parameter to manage, + // and it doesn't hurt to always do the filtering. The only reason + // it exists here is so that we don't always filter when testing locally. + // This is because the filter functionality updates the ComponentFactory.js file + // which could be accidentally committed if run locally. + USE_FILTER_BRANCH: true, GATSBY_BASE_URL: this._config.get<string>('gatsbyBaseUrl'), PREVIEW_BUILD_ENABLED: this._config.get<string>('previewBuildEnabled'), GATSBY_TEST_SEARCH_UI: this._config.get<string>('featureFlagSearchUI'), diff --git a/tests/unit/job/productionJobHandler.test.ts b/tests/unit/job/productionJobHandler.test.ts index 8d07630f2..ace7a81db 100644 --- a/tests/unit/job/productionJobHandler.test.ts +++ b/tests/unit/job/productionJobHandler.test.ts @@ -166,12 +166,6 @@ describe('ProductionJobHandler Tests', () => { jobHandlerTestHelper.setupForSuccess(); await jobHandlerTestHelper.jobHandler.execute(); jobHandlerTestHelper.verifyNextGenSuccess(); - // TODO: Correct number of arguments - expect(jobHandlerTestHelper.fileSystemServices.writeToFile).toBeCalledWith( - `repos/${jobHandlerTestHelper.job.payload.repoName}/.env.production`, - TestDataProvider.getEnvVarsWithPathPrefixWithFlags(jobHandlerTestHelper.job), - { encoding: 'utf8', flag: 'w' } - ); }); test('Default production deploy does not kick off manifest generation', async () => { @@ -244,12 +238,6 @@ describe('ProductionJobHandler Tests', () => { jobHandlerTestHelper.setupForSuccess(); await jobHandlerTestHelper.jobHandler.execute(); jobHandlerTestHelper.verifyNextGenSuccess(); - - expect(jobHandlerTestHelper.fileSystemServices.writeToFile).toBeCalledWith( - `repos/${jobHandlerTestHelper.job.payload.repoName}/.env.production`, - `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${jobHandlerTestHelper.job.payload.branchName}\nPATH_PREFIX=/\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\nGATSBY_MARIAN_URL=test-url\n`, - { encoding: 'utf8', flag: 'w' } - ); }); test('Execute Next Gen Build successfully', async () => {