From 69d9bf8c4e9418ced238b4c41a2c805c7f3c4b2d Mon Sep 17 00:00:00 2001 From: Adam Fowler Date: Wed, 3 Mar 2021 12:11:59 +0000 Subject: [PATCH] Paginate input token test (#424) * Added new version of paginate that includes inputvkey * Fix paginate tests * Return from paginate when input/output token are equal * Undeprecate paginate with moreResults key * Added version of paginate(inputKey:outputKey) without Equatable test * Update AWSClient+Paginate.swift * swift format * Copyright (c) 2017-2021 --- Sources/SotoCore/AWSClient+Paginate.swift | 107 +++++++++++++++++++--- Tests/SotoCoreTests/PaginateTests.swift | 18 ++-- 2 files changed, 102 insertions(+), 23 deletions(-) diff --git a/Sources/SotoCore/AWSClient+Paginate.swift b/Sources/SotoCore/AWSClient+Paginate.swift index fbbe120967..495aebc849 100644 --- a/Sources/SotoCore/AWSClient+Paginate.swift +++ b/Sources/SotoCore/AWSClient+Paginate.swift @@ -2,7 +2,7 @@ // // This source file is part of the Soto for AWS open source project // -// Copyright (c) 2017-2020 the Soto project authors +// Copyright (c) 2017-2021 the Soto project authors // Licensed under Apache License v2.0 // // See LICENSE.txt for license information @@ -23,6 +23,85 @@ public protocol AWSPaginateToken: AWSShape { } extension AWSClient { + /// If an AWS command is returning an arbituary sized array sometimes it adds support for paginating this array + /// ie it will return the array in blocks of a defined size, each block also includes a token which can be used to access + /// the next block. This function loads each block and calls a closure with each block as parameter. This function returns + /// the result of combining all of these blocks using the given closure, + /// + /// - Parameters: + /// - input: Input for request + /// - initialValue: The value to use as the initial accumulating value. `initialValue` is passed to `onPage` the first time it is called. + /// - command: Command to be paginated + /// - inputKey: The name of token in the request object to continue pagination + /// - outputKey: The name of token in the response object to continue pagination + /// - eventLoop: EventLoop to run this process on + /// - logger: Logger used for logging + /// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned + /// along with a boolean indicating if the paginate operation should continue. + public func paginate( + input: Input, + initialValue: Result, + command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture, + inputKey: KeyPath, + outputKey: KeyPath, + logger: Logger = AWSClient.loggingDisabled, + on eventLoop: EventLoop? = nil, + onPage: @escaping (Result, Output, EventLoop) -> EventLoopFuture<(Bool, Result)> + ) -> EventLoopFuture where Input.Token: Equatable { + let eventLoop = eventLoop ?? eventLoopGroup.next() + let promise = eventLoop.makePromise(of: Result.self) + + func paginatePart(input: Input, currentValue: Result) { + let responseFuture = command(input, logger, eventLoop) + .flatMap { response in + return onPage(currentValue, response, eventLoop) + .map { continuePaginate, result -> Void in + guard continuePaginate == true else { return promise.succeed(result) } + // get next block token and construct a new input with this token + guard let outputToken = response[keyPath: outputKey] else { return promise.succeed(result) } + // if output token is still the same as the input token then exit with success + guard outputToken != input[keyPath: inputKey] else { return promise.succeed(result) } + + let input = input.usingPaginationToken(outputToken) + paginatePart(input: input, currentValue: result) + } + } + responseFuture.whenFailure { error in + promise.fail(error) + } + } + + paginatePart(input: input, currentValue: initialValue) + + return promise.futureResult + } + + /// If an AWS command is returning an arbituary sized array sometimes it adds support for paginating this array + /// ie it will return the array in blocks of a defined size, each block also includes a token which can be used to access + /// the next block. This function loads each block and calls a closure with each block as parameter. + /// + /// - Parameters: + /// - input: Input for request + /// - command: Command to be paginated + /// - inputKey: The name of token in the request object to continue pagination + /// - outputKey: The name of token in the response object to continue pagination + /// - eventLoop: EventLoop to run this process on + /// - logger: Logger used for logging + /// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue. + public func paginate( + input: Input, + command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture, + inputKey: KeyPath, + outputKey: KeyPath, + logger: Logger = AWSClient.loggingDisabled, + on eventLoop: EventLoop? = nil, + onPage: @escaping (Output, EventLoop) -> EventLoopFuture + ) -> EventLoopFuture where Input.Token: Equatable { + self.paginate(input: input, initialValue: (), command: command, inputKey: inputKey, outputKey: outputKey, logger: logger, on: eventLoop) { _, output, eventLoop in + return onPage(output, eventLoop).map { rt in (rt, ()) } + } + } + /// If an AWS command is returning an arbituary sized array sometimes it adds support for paginating this array /// ie it will return the array in blocks of a defined size, each block also includes a token which can be used to access /// the next block. This function loads each block and calls a closure with each block as parameter. This function returns @@ -34,7 +113,7 @@ extension AWSClient { /// - command: Command to be paginated /// - tokenKey: The name of token in the response object to continue pagination /// - eventLoop: EventLoop to run this process on - /// - logger: Logger used flot logging + /// - logger: Logger used for logging /// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned /// along with a boolean indicating if the paginate operation should continue. public func paginate( @@ -81,7 +160,7 @@ extension AWSClient { /// - command: Command to be paginated /// - tokenKey: The name of token in the response object to continue pagination /// - eventLoop: EventLoop to run this process on - /// - logger: Logger used flot logging + /// - logger: Logger used for logging /// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue. public func paginate( input: Input, @@ -108,7 +187,7 @@ extension AWSClient { /// - tokenKey: The name of token in the response object to continue pagination /// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data /// - eventLoop: EventLoop to run this process on - /// - logger: Logger used flot logging + /// - logger: Logger used for logging /// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned /// along with a boolean indicating if the paginate operation should continue. public func paginate( @@ -116,7 +195,7 @@ extension AWSClient { initialValue: Result, command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture, tokenKey: KeyPath, - moreResultsKey: KeyPath, + moreResultsKey: KeyPath, logger: Logger = AWSClient.loggingDisabled, on eventLoop: EventLoop? = nil, onPage: @escaping (Result, Output, EventLoop) -> EventLoopFuture<(Bool, Result)> @@ -132,7 +211,7 @@ extension AWSClient { guard continuePaginate == true else { return promise.succeed(result) } // get next block token and construct a new input with this token guard let token = response[keyPath: tokenKey], - response[keyPath: moreResultsKey] else { return promise.succeed(result) } + response[keyPath: moreResultsKey] == true else { return promise.succeed(result) } let input = input.usingPaginationToken(token) paginatePart(input: input, currentValue: result) @@ -158,13 +237,13 @@ extension AWSClient { /// - tokenKey: The name of token in the response object to continue pagination /// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data /// - eventLoop: EventLoop to run this process on - /// - logger: Logger used flot logging + /// - logger: Logger used for logging /// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue. public func paginate( input: Input, command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture, tokenKey: KeyPath, - moreResultsKey: KeyPath, + moreResultsKey: KeyPath, logger: Logger = AWSClient.loggingDisabled, on eventLoop: EventLoop? = nil, onPage: @escaping (Output, EventLoop) -> EventLoopFuture @@ -186,15 +265,16 @@ extension AWSClient { /// - tokenKey: The name of token in the response object to continue pagination /// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data /// - eventLoop: EventLoop to run this process on - /// - logger: Logger used flot logging + /// - logger: Logger used for logging /// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned /// along with a boolean indicating if the paginate operation should continue. + @available(*, deprecated, message: "Deprecated this version of paginate in favour of version that includes the inputKey") public func paginate( input: Input, initialValue: Result, command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture, tokenKey: KeyPath, - moreResultsKey: KeyPath, + moreResultsKey: KeyPath, logger: Logger = AWSClient.loggingDisabled, on eventLoop: EventLoop? = nil, onPage: @escaping (Result, Output, EventLoop) -> EventLoopFuture<(Bool, Result)> @@ -210,7 +290,7 @@ extension AWSClient { guard continuePaginate == true else { return promise.succeed(result) } // get next block token and construct a new input with this token guard let token = response[keyPath: tokenKey], - response[keyPath: moreResultsKey] == true else { return promise.succeed(result) } + response[keyPath: moreResultsKey] else { return promise.succeed(result) } let input = input.usingPaginationToken(token) paginatePart(input: input, currentValue: result) @@ -236,13 +316,14 @@ extension AWSClient { /// - tokenKey: The name of token in the response object to continue pagination /// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data /// - eventLoop: EventLoop to run this process on - /// - logger: Logger used flot logging + /// - logger: Logger used for logging /// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue. + @available(*, deprecated, message: "Deprecated this version of paginate in favour of version that includes the inputKey") public func paginate( input: Input, command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture, tokenKey: KeyPath, - moreResultsKey: KeyPath, + moreResultsKey: KeyPath, logger: Logger = AWSClient.loggingDisabled, on eventLoop: EventLoop? = nil, onPage: @escaping (Output, EventLoop) -> EventLoopFuture diff --git a/Tests/SotoCoreTests/PaginateTests.swift b/Tests/SotoCoreTests/PaginateTests.swift index bacf099109..d7aafc100e 100644 --- a/Tests/SotoCoreTests/PaginateTests.swift +++ b/Tests/SotoCoreTests/PaginateTests.swift @@ -2,7 +2,7 @@ // // This source file is part of the Soto for AWS open source project // -// Copyright (c) 2017-2020 the Soto project authors +// Copyright (c) 2017-2021 the Soto project authors // Licensed under Apache License v2.0 // // See LICENSE.txt for license information @@ -142,14 +142,12 @@ class PaginateTests: XCTestCase { struct StringListOutput: AWSDecodableShape, Encodable { let array: [String] let outputToken: String? - let moreResults: Bool? } // conform to Encodable so server can encode these struct StringList2Output: AWSDecodableShape, Encodable { let array: [String] let outputToken: String? - let moreResults: Bool } func stringList(_ input: StringListInput, logger: Logger, on eventLoop: EventLoop? = nil) -> EventLoopFuture { @@ -168,8 +166,8 @@ class PaginateTests: XCTestCase { return self.client.paginate( input: input, command: self.stringList, - tokenKey: \StringListOutput.outputToken, - moreResultsKey: \StringListOutput.moreResults, + inputKey: \StringListInput.inputToken, + outputKey: \StringListOutput.outputToken, logger: TestEnvironment.logger, on: eventLoop, onPage: onPage @@ -193,8 +191,8 @@ class PaginateTests: XCTestCase { input: input, initialValue: initialValue, command: self.stringList2, - tokenKey: \StringList2Output.outputToken, - moreResultsKey: \StringList2Output.moreResults, + inputKey: \StringListInput.inputToken, + outputKey: \StringList2Output.outputToken, logger: TestEnvironment.logger, on: eventLoop, onPage: onPage @@ -217,14 +215,14 @@ class PaginateTests: XCTestCase { array.append(self.stringList[i]) } var outputToken: String? - var moreResults: Bool = false var continueProcessing = false if endIndex < self.stringList.count { outputToken = self.stringList[endIndex] - moreResults = true continueProcessing = true + } else { + outputToken = input.inputToken } - let output = StringListOutput(array: array, outputToken: outputToken, moreResults: moreResults) + let output = StringListOutput(array: array, outputToken: outputToken) return .result(output, continueProcessing: continueProcessing) }