Skip to content

Commit

Permalink
Paginate input token test (#424)
Browse files Browse the repository at this point in the history 
* Added new version of paginate that includes inputvkey

* Fix paginate tests

* Return from paginate when input/output token are equal

* Undeprecate paginate with moreResults key

* Added version of paginate(inputKey:outputKey) without Equatable test

* Update AWSClient+Paginate.swift

* swift format

* Copyright (c) 2017-2021
  • Loading branch information
@adam-fowler
adam-fowler authored Mar 3, 2021
1 parent 78e03eb commit 69d9bf8
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 23 deletions.
107 changes: 94 additions & 13 deletions Sources/SotoCore/AWSClient+Paginate.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//
// This source file is part of the Soto for AWS open source project
//
// Copyright (c) 2017-2020 the Soto project authors
// Copyright (c) 2017-2021 the Soto project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
Expand All @@ -23,6 +23,85 @@ public protocol AWSPaginateToken: AWSShape {
}

extension AWSClient {
/// If an AWS command is returning an arbituary sized array sometimes it adds support for paginating this array
/// ie it will return the array in blocks of a defined size, each block also includes a token which can be used to access
/// the next block. This function loads each block and calls a closure with each block as parameter. This function returns
/// the result of combining all of these blocks using the given closure,
///
/// - Parameters:
/// - input: Input for request
/// - initialValue: The value to use as the initial accumulating value. `initialValue` is passed to `onPage` the first time it is called.
/// - command: Command to be paginated
/// - inputKey: The name of token in the request object to continue pagination
/// - outputKey: The name of token in the response object to continue pagination
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned
/// along with a boolean indicating if the paginate operation should continue.
public func paginate<Input: AWSPaginateToken, Output: AWSShape, Result>(
input: Input,
initialValue: Result,
command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture<Output>,
inputKey: KeyPath<Input, Input.Token?>,
outputKey: KeyPath<Output, Input.Token?>,
logger: Logger = AWSClient.loggingDisabled,
on eventLoop: EventLoop? = nil,
onPage: @escaping (Result, Output, EventLoop) -> EventLoopFuture<(Bool, Result)>
) -> EventLoopFuture<Result> where Input.Token: Equatable {
let eventLoop = eventLoop ?? eventLoopGroup.next()
let promise = eventLoop.makePromise(of: Result.self)

func paginatePart(input: Input, currentValue: Result) {
let responseFuture = command(input, logger, eventLoop)
.flatMap { response in
return onPage(currentValue, response, eventLoop)
.map { continuePaginate, result -> Void in
guard continuePaginate == true else { return promise.succeed(result) }
// get next block token and construct a new input with this token
guard let outputToken = response[keyPath: outputKey] else { return promise.succeed(result) }
// if output token is still the same as the input token then exit with success
guard outputToken != input[keyPath: inputKey] else { return promise.succeed(result) }

let input = input.usingPaginationToken(outputToken)
paginatePart(input: input, currentValue: result)
}
}
responseFuture.whenFailure { error in
promise.fail(error)
}
}

paginatePart(input: input, currentValue: initialValue)

return promise.futureResult
}

/// If an AWS command is returning an arbituary sized array sometimes it adds support for paginating this array
/// ie it will return the array in blocks of a defined size, each block also includes a token which can be used to access
/// the next block. This function loads each block and calls a closure with each block as parameter.
///
/// - Parameters:
/// - input: Input for request
/// - command: Command to be paginated
/// - inputKey: The name of token in the request object to continue pagination
/// - outputKey: The name of token in the response object to continue pagination
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue.
public func paginate<Input: AWSPaginateToken, Output: AWSShape>(
input: Input,
command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture<Output>,
inputKey: KeyPath<Input, Input.Token?>,
outputKey: KeyPath<Output, Input.Token?>,
logger: Logger = AWSClient.loggingDisabled,
on eventLoop: EventLoop? = nil,
onPage: @escaping (Output, EventLoop) -> EventLoopFuture<Bool>
) -> EventLoopFuture<Void> where Input.Token: Equatable {
self.paginate(input: input, initialValue: (), command: command, inputKey: inputKey, outputKey: outputKey, logger: logger, on: eventLoop) { _, output, eventLoop in
return onPage(output, eventLoop).map { rt in (rt, ()) }
}
}

/// If an AWS command is returning an arbituary sized array sometimes it adds support for paginating this array
/// ie it will return the array in blocks of a defined size, each block also includes a token which can be used to access
/// the next block. This function loads each block and calls a closure with each block as parameter. This function returns
Expand All @@ -34,7 +113,7 @@ extension AWSClient {
/// - command: Command to be paginated
/// - tokenKey: The name of token in the response object to continue pagination
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used flot logging
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned
/// along with a boolean indicating if the paginate operation should continue.
public func paginate<Input: AWSPaginateToken, Output: AWSShape, Result>(
Expand Down Expand Up @@ -81,7 +160,7 @@ extension AWSClient {
/// - command: Command to be paginated
/// - tokenKey: The name of token in the response object to continue pagination
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used flot logging
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue.
public func paginate<Input: AWSPaginateToken, Output: AWSShape>(
input: Input,
Expand All @@ -108,15 +187,15 @@ extension AWSClient {
/// - tokenKey: The name of token in the response object to continue pagination
/// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used flot logging
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned
/// along with a boolean indicating if the paginate operation should continue.
public func paginate<Input: AWSPaginateToken, Output: AWSShape, Result>(
input: Input,
initialValue: Result,
command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture<Output>,
tokenKey: KeyPath<Output, Input.Token?>,
moreResultsKey: KeyPath<Output, Bool>,
moreResultsKey: KeyPath<Output, Bool?>,
logger: Logger = AWSClient.loggingDisabled,
on eventLoop: EventLoop? = nil,
onPage: @escaping (Result, Output, EventLoop) -> EventLoopFuture<(Bool, Result)>
Expand All @@ -132,7 +211,7 @@ extension AWSClient {
guard continuePaginate == true else { return promise.succeed(result) }
// get next block token and construct a new input with this token
guard let token = response[keyPath: tokenKey],
response[keyPath: moreResultsKey] else { return promise.succeed(result) }
response[keyPath: moreResultsKey] == true else { return promise.succeed(result) }

let input = input.usingPaginationToken(token)
paginatePart(input: input, currentValue: result)
Expand All @@ -158,13 +237,13 @@ extension AWSClient {
/// - tokenKey: The name of token in the response object to continue pagination
/// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used flot logging
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue.
public func paginate<Input: AWSPaginateToken, Output: AWSShape>(
input: Input,
command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture<Output>,
tokenKey: KeyPath<Output, Input.Token?>,
moreResultsKey: KeyPath<Output, Bool>,
moreResultsKey: KeyPath<Output, Bool?>,
logger: Logger = AWSClient.loggingDisabled,
on eventLoop: EventLoop? = nil,
onPage: @escaping (Output, EventLoop) -> EventLoopFuture<Bool>
Expand All @@ -186,15 +265,16 @@ extension AWSClient {
/// - tokenKey: The name of token in the response object to continue pagination
/// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used flot logging
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. It combines an accumulating result with the contents of response from the call to AWS. This combined result is then returned
/// along with a boolean indicating if the paginate operation should continue.
@available(*, deprecated, message: "Deprecated this version of paginate in favour of version that includes the inputKey")
public func paginate<Input: AWSPaginateToken, Output: AWSShape, Result>(
input: Input,
initialValue: Result,
command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture<Output>,
tokenKey: KeyPath<Output, Input.Token?>,
moreResultsKey: KeyPath<Output, Bool?>,
moreResultsKey: KeyPath<Output, Bool>,
logger: Logger = AWSClient.loggingDisabled,
on eventLoop: EventLoop? = nil,
onPage: @escaping (Result, Output, EventLoop) -> EventLoopFuture<(Bool, Result)>
Expand All @@ -210,7 +290,7 @@ extension AWSClient {
guard continuePaginate == true else { return promise.succeed(result) }
// get next block token and construct a new input with this token
guard let token = response[keyPath: tokenKey],
response[keyPath: moreResultsKey] == true else { return promise.succeed(result) }
response[keyPath: moreResultsKey] else { return promise.succeed(result) }

let input = input.usingPaginationToken(token)
paginatePart(input: input, currentValue: result)
Expand All @@ -236,13 +316,14 @@ extension AWSClient {
/// - tokenKey: The name of token in the response object to continue pagination
/// - moreResultsKey: The KeyPath for the member of the output that indicates whether we should ask for more data
/// - eventLoop: EventLoop to run this process on
/// - logger: Logger used flot logging
/// - logger: Logger used for logging
/// - onPage: closure called with each block of entries. Returns boolean indicating whether we should continue.
@available(*, deprecated, message: "Deprecated this version of paginate in favour of version that includes the inputKey")
public func paginate<Input: AWSPaginateToken, Output: AWSShape>(
input: Input,
command: @escaping (Input, Logger, EventLoop?) -> EventLoopFuture<Output>,
tokenKey: KeyPath<Output, Input.Token?>,
moreResultsKey: KeyPath<Output, Bool?>,
moreResultsKey: KeyPath<Output, Bool>,
logger: Logger = AWSClient.loggingDisabled,
on eventLoop: EventLoop? = nil,
onPage: @escaping (Output, EventLoop) -> EventLoopFuture<Bool>
Expand Down
18 changes: 8 additions & 10 deletions Tests/SotoCoreTests/PaginateTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//
// This source file is part of the Soto for AWS open source project
//
// Copyright (c) 2017-2020 the Soto project authors
// Copyright (c) 2017-2021 the Soto project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
Expand Down Expand Up @@ -142,14 +142,12 @@ class PaginateTests: XCTestCase {
struct StringListOutput: AWSDecodableShape, Encodable {
let array: [String]
let outputToken: String?
let moreResults: Bool?
}

// conform to Encodable so server can encode these
struct StringList2Output: AWSDecodableShape, Encodable {
let array: [String]
let outputToken: String?
let moreResults: Bool
}

func stringList(_ input: StringListInput, logger: Logger, on eventLoop: EventLoop? = nil) -> EventLoopFuture<StringListOutput> {
Expand All @@ -168,8 +166,8 @@ class PaginateTests: XCTestCase {
return self.client.paginate(
input: input,
command: self.stringList,
tokenKey: \StringListOutput.outputToken,
moreResultsKey: \StringListOutput.moreResults,
inputKey: \StringListInput.inputToken,
outputKey: \StringListOutput.outputToken,
logger: TestEnvironment.logger,
on: eventLoop,
onPage: onPage
Expand All @@ -193,8 +191,8 @@ class PaginateTests: XCTestCase {
input: input,
initialValue: initialValue,
command: self.stringList2,
tokenKey: \StringList2Output.outputToken,
moreResultsKey: \StringList2Output.moreResults,
inputKey: \StringListInput.inputToken,
outputKey: \StringList2Output.outputToken,
logger: TestEnvironment.logger,
on: eventLoop,
onPage: onPage
Expand All @@ -217,14 +215,14 @@ class PaginateTests: XCTestCase {
array.append(self.stringList[i])
}
var outputToken: String?
var moreResults: Bool = false
var continueProcessing = false
if endIndex < self.stringList.count {
outputToken = self.stringList[endIndex]
moreResults = true
continueProcessing = true
} else {
outputToken = input.inputToken
}
let output = StringListOutput(array: array, outputToken: outputToken, moreResults: moreResults)
let output = StringListOutput(array: array, outputToken: outputToken)
return .result(output, continueProcessing: continueProcessing)
}

Expand Down

0 comments on commit 69d9bf8

Please sign in to comment.