diff --git a/README.md b/README.md index f1a544115..e6f98f28c 100644 --- a/README.md +++ b/README.md @@ -30,15 +30,28 @@ If you have previously set up a v1 developer portal (non-SAM deployed), you will #### Deploy Run: + >In the command below, replace the `your-lambda-artifacts-bucket-name` with the name of a bucket that you manage and that already exists. Then, run: + ```bash -sam package --template-file ./cloudformation/template.yaml --output-template-file ./cloudformation/packaged.yaml --s3-bucket your-lambda-artifacts-bucket-name +sam package --template-file ./cloudformation/template.yaml \ + --output-template-file ./cloudformation/packaged.yaml \ + --s3-bucket your-lambda-artifacts-bucket-name ``` -Then run: +Then run: + >In the command below, replace the `your-lambda-artifacts-bucket-name` with the name of a bucket that you manage and that already exists, and replace `custom-prefix` with some prefix that is globally unique, like your org name or username. Then, run: + ```bash -sam deploy --template-file ./cloudformation/packaged.yaml --stack-name "dev-portal" --s3-bucket your-lambda-artifacts-bucket-name --capabilities CAPABILITY_NAMED_IAM --parameter-overrides DevPortalSiteS3BucketName="custom-prefix-dev-portal-static-assets" ArtifactsS3BucketName="custom-prefix-dev-portal-artifacts" CognitoDomainNameOrPrefix="custom-prefix" +sam deploy --template-file ./cloudformation/packaged.yaml \ + --stack-name "dev-portal" \ + --s3-bucket your-lambda-artifacts-bucket-name \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides \ + DevPortalSiteS3BucketName="custom-prefix-dev-portal-static-assets" \ + ArtifactsS3BucketName="custom-prefix-dev-portal-artifacts" \ + CognitoDomainNameOrPrefix="custom-prefix" ``` The command will exit when the stack creation is successful. If you'd like to watch it create in real-time, you can log into the cloudformation console. @@ -46,7 +59,8 @@ The command will exit when the stack creation is successful. If you'd like to wa To get the URL for the newly created developer portal instance, find the websiteURL field in the cloudformation console's outputs or run this command: ```bash -aws cloudformation describe-stacks --query "Stacks[?StackName=='dev-portal'][Outputs[?OutputKey=='WebsiteURL']][][].OutputValue" +aws cloudformation describe-stacks --query \ + "Stacks[?StackName=='dev-portal'][Outputs[?OutputKey=='WebsiteURL']][][].OutputValue" ``` You can override any of the parameters in the template using the `--parameter-overrides key="value"` format. This will be necessary if you intend to deploy several instances of the developer portal or customize some of the features. You can see a full list of overridable parameters in `cloudformation/template.yaml` under the `Parameters` section. diff --git a/cloudformation/template.yaml b/cloudformation/template.yaml index 6242f4138..789c43e33 100644 --- a/cloudformation/template.yaml +++ b/cloudformation/template.yaml @@ -74,6 +74,10 @@ Parameters: Type: String Description: By default, a static asset rebuild doesn't overwrite custom-content. Provide the value `overwrite-content` to replace the custom-content with your local version. Don't do this unless you know what you're doing -- all custom changes in your s3 bucket will be lost. Default: '' + AllowedValues: + - 'overwrite-content' + - '' + ConstraintDescription: Malformed input - Parameter StaticAssetRebuildMode value must be either 'overwrite-content' or left blank. MarketplaceSubscriptionTopicProductCode: Type: String @@ -108,11 +112,19 @@ Parameters: Type: String Description: Only applicable if creating a custom domain name for your dev portal. Defaults to false, and you'll need to provide your own nameserver hosting. If set to true, a Route53 HostedZone and RecordSet are created for you. Default: 'false' + AllowedValues: + - 'false' + - 'true' + ConstraintDescription: Malformed input - Parameter UseRoute53Nameservers value must be either 'true' or 'false' DevelopmentMode: Type: String Description: Enabling this weakens security features (OAI, SSL, site S3 bucket with public read ACLs, Cognito callback verification, CORS, etc.) for easier development. It also breaks frontend routing (except to /index.html), including deep linking and page refresh. Do not enable this in production! Additionally, do not update a stack that was previously in development mode to be a production stack; instead, make a new stack that has never been in development mode. Default: 'false' + AllowedValues: + - 'false' + - 'true' + ConstraintDescription: Malformed input - Parameter DevelopmentMode value must be either 'true' or 'false' Conditions: UseCustomDomainName: !And [!And [!Not [!Equals [!Ref CustomDomainName, '']], !Not [!Equals [!Ref CustomDomainNameAcmCertArn, '']]], !Condition NotDevelopmentMode] diff --git a/lambdas/backend/_common/customers-controller.js b/lambdas/backend/_common/customers-controller.js index 82f7509b8..7f2021e62 100644 --- a/lambdas/backend/_common/customers-controller.js +++ b/lambdas/backend/_common/customers-controller.js @@ -3,6 +3,7 @@ 'use strict' const AWS = require('aws-sdk') +const { getAllUsagePlans } = require('../shared/get-all-usage-plans') const dynamoDb = new AWS.DynamoDB.DocumentClient() const apigateway = new AWS.APIGateway() @@ -195,10 +196,9 @@ function getUsagePlansForCustomer(cognitoIdentityId, error, callback) { keyId, limit: 1000 } - apigateway.getUsagePlans(params, (err, usagePlansData) => { - if (err) error(err) - else callback(usagePlansData) - }) + getAllUsagePlans(apigateway, params) + .then(usagePlansData => callback({ items: usagePlansData })) + .catch(err => error(err)) } }) } @@ -209,26 +209,22 @@ function getUsagePlanForProductCode(productCode, error, callback) { // do a linear scan of usage plans for name matching productCode var params = { limit: 1000 - }; - apigateway.getUsagePlans(params, function(err, data) { - if (err) { - error(err) - } else { - console.log(`Got usage plans ${JSON.stringify(data.items)}`) + } + getAllUsagePlans(apigateway, params).then(usagePlans => { + console.log(`Got usage plans ${JSON.stringify(usagePlans)}`) - // note: ensure that only one usage plan maps to a given marketplace product code - const usageplan = data.items.find(function (item) { - return item.productCode !== undefined && item.productCode === productCode - }) - if (usageplan !== undefined) { - console.log(`Found usage plan matching ${productCode}`) - callback(usageplan) - } else { - console.log(`Couldn't find usageplan matching product code ${productCode}`) - error(`Couldn't find usageplan matching product code ${productCode}`) - } + // note: ensure that only one usage plan maps to a given marketplace product code + const usageplan = usagePlans.find(function (item) { + return item.productCode !== undefined && item.productCode === productCode + }) + if (usageplan !== undefined) { + console.log(`Found usage plan matching ${productCode}`) + callback(usageplan) + } else { + console.log(`Couldn't find usageplan matching product code ${productCode}`) + error(`Couldn't find usageplan matching product code ${productCode}`) } - }); + }).catch(err => error(err)) } function updateCustomerMarketplaceId(cognitoIdentityId, marketplaceCustomerId, error, success) { @@ -323,16 +319,6 @@ function updateCustomerApiKeyId(cognitoIdentityId, apiKeyId, error, success) { }) } -// function getUsagePlans(error, callback) { -// const params = { -// limit: 1000 -// } -// apigateway.getUsagePlans(params, (err, data) => { -// if (err) error(err) -// else callback(data) -// }) -// } - module.exports = { ensureCustomerItem, subscribe, diff --git a/lambdas/backend/express-route-handlers.js b/lambdas/backend/express-route-handlers.js index f989c6e70..f65c33988 100644 --- a/lambdas/backend/express-route-handlers.js +++ b/lambdas/backend/express-route-handlers.js @@ -3,6 +3,7 @@ const feedbackController = require('./_common/feedback-controller.js') const AWS = require('aws-sdk') const catalog = require('./catalog/index') const hash = require('object-hash') +const { getAllUsagePlans } = require('./shared/get-all-usage-plans') const Datauri = require('datauri') @@ -421,7 +422,7 @@ async function getAdminCatalogVisibility(req, res) { }) }) - let usagePlans = await exports.apigateway.getUsagePlans().promise() + let usagePlans = await getAllUsagePlans(exports.apigateway) // In the case of apiGateway APIs, the client doesn't know if there are usage plan associated or not // so we need to provide that information. This can't be merged with the above loop: @@ -431,7 +432,7 @@ async function getAdminCatalogVisibility(req, res) { visibility.apiGateway.map((apiEntry) => { apiEntry.subscribable = false - usagePlans.items.forEach((usagePlan) => { + usagePlans.forEach((usagePlan) => { usagePlan.apiStages.forEach((apiStage) => { if(apiEntry.id === apiStage.apiId && apiEntry.stage === apiStage.stage) { apiEntry.subscribable = true diff --git a/lambdas/backend/shared/get-all-usage-plans.js b/lambdas/backend/shared/get-all-usage-plans.js new file mode 100644 index 000000000..c27dbebaf --- /dev/null +++ b/lambdas/backend/shared/get-all-usage-plans.js @@ -0,0 +1,33 @@ +/** + * Fetches all usage plans, combining all pages into a single array. + * + * @param apiGateway + * an instance of `AWS.APIGateway` to use for API calls + * + * @param paramOverrides + * a parameter object passed in calls to `APIGateway.getUsagePlans` + * + * @returns + * a Promise resolving with an array of items returned from + * `APIGateway.getUsagePlans` calls + */ +async function getAllUsagePlans(apiGateway, paramOverrides = {}) { + // The maximum allowed value of `limit` is 500 according to + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/APIGateway.html#getUsagePlans-property + const defaultParams = {limit: 500, ...paramOverrides} + + console.log('Fetching first page of usage plans') + let response = await apiGateway.getUsagePlans(defaultParams).promise() + const usagePlans = response.items + + while (response.position) { + console.log(`Fetching next page of usage plans, at position=[${response.position}]`) + const nextParams = {...defaultParams, position: response.position} + response = await apiGateway.getUsagePlans(nextParams).promise() + usagePlans.push(...response.items) + } + + return usagePlans +} + +module.exports = { getAllUsagePlans } diff --git a/lambdas/catalog-updater/index.js b/lambdas/catalog-updater/index.js index b30b2c221..068066c90 100644 --- a/lambdas/catalog-updater/index.js +++ b/lambdas/catalog-updater/index.js @@ -12,6 +12,8 @@ let AWS = require('aws-sdk'), bucketName = '', hash = require('object-hash') +const { getAllUsagePlans } = require('./shared/get-all-usage-plans') + /** * Takes in an s3 listObjectsV2 object and returns whether it's a "swagger file" (one ending in .JSON, .YAML, or .YML), * and whether it's in the catalog folder (S3 Key starts with "catalog/"). @@ -185,10 +187,9 @@ function buildCatalog(swaggerFiles, sdkGeneration) { generic: [] } - return exports.gateway.getUsagePlans({}).promise() - .then((result) => { - console.log(`usagePlans: ${JSON.stringify(result.items, null, 4)}`) - let usagePlans = result.items + return getAllUsagePlans(exports.gateway) + .then(usagePlans => { + console.log(`usagePlans: ${JSON.stringify(usagePlans, null, 4)}`) for (let i = 0; i < usagePlans.length; i++) { catalog.apiGateway[i] = usagePlanToCatalogObject(usagePlans[i], swaggerFiles, sdkGeneration) } @@ -252,4 +253,4 @@ exports = module.exports = { gateway: new AWS.APIGateway(), handler, hash -} \ No newline at end of file +} diff --git a/lambdas/catalog-updater/shared/get-all-usage-plans.js b/lambdas/catalog-updater/shared/get-all-usage-plans.js new file mode 100644 index 000000000..c27dbebaf --- /dev/null +++ b/lambdas/catalog-updater/shared/get-all-usage-plans.js @@ -0,0 +1,33 @@ +/** + * Fetches all usage plans, combining all pages into a single array. + * + * @param apiGateway + * an instance of `AWS.APIGateway` to use for API calls + * + * @param paramOverrides + * a parameter object passed in calls to `APIGateway.getUsagePlans` + * + * @returns + * a Promise resolving with an array of items returned from + * `APIGateway.getUsagePlans` calls + */ +async function getAllUsagePlans(apiGateway, paramOverrides = {}) { + // The maximum allowed value of `limit` is 500 according to + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/APIGateway.html#getUsagePlans-property + const defaultParams = {limit: 500, ...paramOverrides} + + console.log('Fetching first page of usage plans') + let response = await apiGateway.getUsagePlans(defaultParams).promise() + const usagePlans = response.items + + while (response.position) { + console.log(`Fetching next page of usage plans, at position=[${response.position}]`) + const nextParams = {...defaultParams, position: response.position} + response = await apiGateway.getUsagePlans(nextParams).promise() + usagePlans.push(...response.items) + } + + return usagePlans +} + +module.exports = { getAllUsagePlans } diff --git a/lambdas/shared/__tests__/get-all-usage-plans-test.js b/lambdas/shared/__tests__/get-all-usage-plans-test.js new file mode 100644 index 000000000..77ac7c16a --- /dev/null +++ b/lambdas/shared/__tests__/get-all-usage-plans-test.js @@ -0,0 +1,106 @@ +const { getAllUsagePlans } = require('../get-all-usage-plans') + +const promiser = require('../../setup-jest').promiser + +const mockUsagePlanItem = () => ({ + id: '1a2b3c', + name: '1a2b3c', + apiStages: [ + { + apiId: 'anmlcrckrs', + stage: 'prod', + }, + { + apiId: 'jlpnochips', + stage: 'beta', + }, + ], + throttle: { + burstLimit: 10, + rateLimit: 10, + }, + quota: { + limit: 10000, + offset: 0, + period: 'DAY', + } +}) + +describe('getAllUsagePlans', () => { + test('returns all usage plans, when none exist', async () => { + const mockApiGateway = { + getUsagePlans: jest.fn().mockReturnValueOnce(promiser({ + items: [] + })) + } + + const result = await getAllUsagePlans(mockApiGateway) + const mocked = mockApiGateway.getUsagePlans.mock + expect(mocked.calls.length).toBe(1) + expect(mocked.calls[0][0]).not.toHaveProperty('position') + expect(result).toHaveLength(0) + }) + + test('returns all usage plans, when only one page of usage plans exists', async () => { + const mockApiGateway = { + getUsagePlans: jest.fn().mockReturnValueOnce(promiser({ + items: [ + mockUsagePlanItem(), + mockUsagePlanItem(), + mockUsagePlanItem(), + mockUsagePlanItem(), + ] + })) + } + + const result = await getAllUsagePlans(mockApiGateway) + const mocked = mockApiGateway.getUsagePlans.mock + expect(mocked.calls.length).toBe(1) + expect(mocked.calls[0][0]).not.toHaveProperty('position') + expect(result).toHaveLength(4) + }) + + test('returns all usage plans, when multiple pages of usage plans exist', async () => { + const mockApiGateway = { + getUsagePlans: jest.fn().mockReturnValueOnce(promiser({ + items: [ + mockUsagePlanItem(), + mockUsagePlanItem(), + mockUsagePlanItem(), + mockUsagePlanItem(), + ], + position: 'qwertyuiopasdf%3D%3D', + })).mockReturnValueOnce(promiser({ + items: [ + mockUsagePlanItem(), + mockUsagePlanItem(), + mockUsagePlanItem(), + mockUsagePlanItem(), + ], + position: 'zxcvbnm1234567%3D%3D', + })).mockReturnValueOnce(promiser({ + items: [ + mockUsagePlanItem(), + mockUsagePlanItem(), + ], + })) + } + + const result = await getAllUsagePlans(mockApiGateway) + const mocked = mockApiGateway.getUsagePlans.mock + expect(mocked.calls.length).toBe(3) + expect(mocked.calls[0][0]).not.toHaveProperty('position') + expect(mocked.calls[1][0]).toHaveProperty('position', 'qwertyuiopasdf%3D%3D') + expect(mocked.calls[2][0]).toHaveProperty('position', 'zxcvbnm1234567%3D%3D') + expect(result).toHaveLength(10) + }) + + test('passes through an API Gateway request error', async () => { + const expectedError = {} + const mockApiGateway = { + getUsagePlans: jest.fn().mockReturnValueOnce(promiser(null, expectedError)) + } + + await expect(getAllUsagePlans(mockApiGateway)).rejects.toStrictEqual(expectedError) + }) +}) diff --git a/lambdas/shared/get-all-usage-plans.js b/lambdas/shared/get-all-usage-plans.js new file mode 100644 index 000000000..c27dbebaf --- /dev/null +++ b/lambdas/shared/get-all-usage-plans.js @@ -0,0 +1,33 @@ +/** + * Fetches all usage plans, combining all pages into a single array. + * + * @param apiGateway + * an instance of `AWS.APIGateway` to use for API calls + * + * @param paramOverrides + * a parameter object passed in calls to `APIGateway.getUsagePlans` + * + * @returns + * a Promise resolving with an array of items returned from + * `APIGateway.getUsagePlans` calls + */ +async function getAllUsagePlans(apiGateway, paramOverrides = {}) { + // The maximum allowed value of `limit` is 500 according to + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/APIGateway.html#getUsagePlans-property + const defaultParams = {limit: 500, ...paramOverrides} + + console.log('Fetching first page of usage plans') + let response = await apiGateway.getUsagePlans(defaultParams).promise() + const usagePlans = response.items + + while (response.position) { + console.log(`Fetching next page of usage plans, at position=[${response.position}]`) + const nextParams = {...defaultParams, position: response.position} + response = await apiGateway.getUsagePlans(nextParams).promise() + usagePlans.push(...response.items) + } + + return usagePlans +} + +module.exports = { getAllUsagePlans } diff --git a/package.json b/package.json index 9d3b262d1..7c50271cc 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Serverless Developer Portal for API Gateway", "main": "lambda.js", "scripts": { - "get-dependencies": "npm run get-dev-portal-dependencies && npm run get-lambda-dependencies;", + "get-dependencies": "npm install && npm run get-dev-portal-dependencies && npm run get-lambda-dependencies;", "get-lambda-dependencies": "find lambdas/*/package.json -type f -exec sh -c 'cd $(dirname {}) && npm run get-dependencies' \\;", "get-dev-portal-dependencies": "(cd dev-portal; npm run get-dependencies)", "test": "node scripts/test.js",