Move Benchmark code into soto-core (#388)

* Move Benchmark code into soto-core

* Swift Format

* Update sanity.sh to not fail on Benchmark build files

.github/workflows/benchmark.yml

@@ -13,18 +13,10 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
       with:
-        repository: soto-project/soto-benchmark
-        path: soto-benchmark
         fetch-depth: 1
     - name: Benchmark
       run: |
-        cd soto-benchmark
-        echo ${GITHUB_REF}
-        if [ -z ${GITHUB_REF} ]; then
-          swift package edit soto-core --branch ${GITHUB_REF}
-        else
-          swift package edit soto-core --revision ${GITHUB_SHA}
-        fi
+        cd Benchmark
         swift run -c release
 
   linux:
@@ -40,19 +32,12 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
       with:
-        repository: soto-project/soto-benchmark
-        path: soto-benchmark
         fetch-depth: 1
     - name: Install dependencies
       run: |
         apt-get update -qq
         apt-get install -q -y tzdata zlib1g-dev curl
     - name: Benchmark
       run: |
-        cd soto-benchmark
-        if [ -z ${GITHUB_REF} ]; then
-          swift package edit soto-core --branch ${GITHUB_REF}
-        else
-          swift package edit soto-core --revision ${GITHUB_SHA}
-        fi
+        cd Benchmark
         swift run -c release

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
-/.build
-/.swiftpm
+.build
+.swiftpm
 /Packages
 /*.xcodeproj
 Package.resolved

Benchmark/Package.swift

@@ -0,0 +1,18 @@
+// swift-tools-version:5.2
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "soto-benchmark",
+    dependencies: [
+        .package(url: "https://github.com/soto-project/soto-core", .branch("main")),
+        .package(name: "Benchmark", url: "https://github.com/google/swift-benchmark", .branch("master")),
+    ],
+    targets: [
+        .target(name: "soto-benchmark", dependencies: [
+            .product(name: "SotoCore", package: "soto-core"),
+            .product(name: "Benchmark", package: "Benchmark"),
+        ]),
+    ]
+)

Benchmark/README.md

@@ -0,0 +1,3 @@
+# soto-benchmark
+
+Benchmark testing for soto-core

Benchmark/Sources/soto-benchmark/AWSClientSuite.swift

@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2020 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Dispatch
+import Foundation
+import NIO
+import SotoCore
+
+struct RequestThrowMiddleware: AWSServiceMiddleware {
+    struct Error: Swift.Error {}
+
+    func chain(request: AWSRequest, context: AWSMiddlewareContext) throws -> AWSRequest {
+        _ = request.body.asByteBuffer(byteBufferAllocator: ByteBufferAllocator())
+        throw Error()
+    }
+}
+
+struct HeaderShape: AWSEncodableShape {
+    static let _encoding: [AWSMemberEncoding] = [
+        .init(label: "a", location: .header(locationName: "A")),
+        .init(label: "b", location: .header(locationName: "B")),
+    ]
+    let a: String
+    let b: Int
+}
+
+struct QueryShape: AWSEncodableShape {
+    static let _encoding: [AWSMemberEncoding] = [
+        .init(label: "a", location: .querystring(locationName: "A")),
+        .init(label: "b", location: .querystring(locationName: "B")),
+    ]
+    let a: String
+    let b: Int
+}
+
+struct Shape1: AWSEncodableShape {
+    let a: String
+    let b: Int
+}
+
+struct Shape: AWSEncodableShape {
+    let a: String
+    let b: Int
+    let c: [String]
+    let d: [String: Int]
+}
+
+let awsClientSuite = BenchmarkSuite(name: "AWSClient", settings: Iterations(10000), WarmupIterations(2)) { suite in
+    // time request construction by throwing an error in request middleware. This means waiting on client.execute should
+    // take the amount of time it took to construct the request
+    let client = AWSClient(
+        credentialProvider: .static(accessKeyId: "foo", secretAccessKey: "bar"),
+        middlewares: [RequestThrowMiddleware()],
+        httpClientProvider: .createNew
+    )
+    let jsonService = AWSServiceConfig(
+        region: .useast1, partition: .aws, service: "test-service", serviceProtocol: .json(version: "1.1"), apiVersion: "10-10-2010"
+    )
+    let xmlService = AWSServiceConfig(
+        region: .useast1, partition: .aws, service: "test-service", serviceProtocol: .restxml, apiVersion: "10-10-2010"
+    )
+    let queryService = AWSServiceConfig(
+        region: .useast1, partition: .aws, service: "test-service", serviceProtocol: .query, apiVersion: "10-10-2010"
+    )
+
+    suite.benchmark("empty-request") {
+        try? client.execute(operation: "TestOperation", path: "/", httpMethod: .GET, serviceConfig: jsonService).wait()
+    }
+
+    let headerInput = HeaderShape(a: "TestString", b: 345_348)
+    suite.benchmark("header-request") {
+        try? client.execute(operation: "TestOperation", path: "/", httpMethod: .GET, serviceConfig: jsonService, input: headerInput).wait()
+    }
+
+    let queryInput = QueryShape(a: "TestString", b: 345_348)
+    suite.benchmark("querystring-request") {
+        try? client.execute(operation: "TestOperation", path: "/", httpMethod: .GET, serviceConfig: jsonService, input: queryInput).wait()
+    }
+
+    // test json, xml and query generation timing
+    let input = Shape(a: "TestString", b: 345_348, c: ["one", "two", "three"], d: ["one": 1, "two": 2, "three": 3])
+    suite.benchmark("json-request") {
+        try? client.execute(operation: "TestOperation", path: "/", httpMethod: .GET, serviceConfig: jsonService, input: input).wait()
+    }
+    suite.benchmark("xml-request") {
+        try? client.execute(operation: "TestOperation", path: "/", httpMethod: .GET, serviceConfig: xmlService, input: input).wait()
+    }
+    suite.benchmark("query-request") {
+        try? client.execute(operation: "TestOperation", path: "/", httpMethod: .GET, serviceConfig: queryService, input: input).wait()
+    }
+}

Benchmark/Sources/soto-benchmark/AWSSignerV4Suite.swift

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2020 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Foundation
+import SotoSignerV4
+
+let awsSignerV4Suite = BenchmarkSuite(name: "AWSSignerV4", settings: Iterations(1000), WarmupIterations(2)) { suite in
+    let string = "testing, testing, 1,2,1,2"
+    let credentials: Credential = StaticCredential(accessKeyId: "MYACCESSKEY", secretAccessKey: "MYSECRETACCESSKEY")
+    let signer = AWSSigner(credentials: credentials, name: "s3", region: "eu-west-1")
+
+    suite.benchmark("sign-headers") {
+        _ = signer.signHeaders(url: URL(string: "https://test-bucket.s3.amazonaws.com/test-put.txt")!, method: .GET, headers: ["Content-Type": "application/x-www-form-urlencoded; charset=utf-8"], body: .string(string))
+    }
+
+    suite.benchmark("sign-url") {
+        _ = signer.signURL(url: URL(string: "https://test-bucket.s3.amazonaws.com/test-put.txt")!, method: .GET, body: .string(string), expires: .hours(1))
+    }
+}

Benchmark/Sources/soto-benchmark/EncoderSuites.swift

@@ -0,0 +1,117 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2020 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Foundation
+import SotoCore
+import SotoXML
+
+protocol EncoderProtocol {
+    associatedtype Output
+    func encode<Input: Encodable>(_ type: Input) throws -> Output
+}
+
+extension XMLEncoder: EncoderProtocol {
+    typealias Output = XML.Element
+    func encode<Input: Encodable>(_ value: Input) throws -> Output {
+        try self.encode(value, name: "BenchmarkTest")
+    }
+}
+
+extension QueryEncoder: EncoderProtocol {
+    typealias Output = String?
+    func encode<Input: Encodable>(_ value: Input) throws -> Output {
+        try self.encode(value, name: "BenchmarkTest")
+    }
+}
+
+struct Numbers: Codable {
+    let b: Bool
+    let i: Int
+    let f: Float
+    let d: Double
+}
+
+struct Strings: Codable {
+    let s: String
+    let s2: String?
+}
+
+struct Arrays: Codable {
+    let a1: [Int]
+    let a2: [String]
+}
+
+struct Dictionaries: Codable {
+    let d: [String: Int]
+}
+
+/// Generic suite of benchmark tests for an Encoder.
+func encoderSuite<E: EncoderProtocol>(for encoder: E, suite: BenchmarkSuite) {
+    let numbers = Numbers(b: true, i: 3478, f: 34.4633, d: 9585)
+    suite.benchmark("numbers") {
+        _ = try encoder.encode(numbers)
+    }
+
+    let strings = Strings(s: "Benchmark string", s2: "optional")
+    suite.benchmark("strings") {
+        _ = try encoder.encode(strings)
+    }
+
+    let arrays = Arrays(a1: [234, 23, 55, 1], a2: ["Benchmark", "string"])
+    suite.benchmark("arrays") {
+        _ = try encoder.encode(arrays)
+    }
+
+    let dictionaries = Dictionaries(d: ["benchmark": 1, "tests": 2, "again": 6])
+    suite.benchmark("dictionaries") {
+        _ = try encoder.encode(dictionaries)
+    }
+}
+
+/// Suite of benchmark tests for XMLEncoder
+let xmlEncoderSuite = BenchmarkSuite(name: "XMLEncoder", settings: Iterations(10000), WarmupIterations(10)) { suite in
+    encoderSuite(for: XMLEncoder(), suite: suite)
+}
+
+/// Suite of benchmark tests to XMLDecoder
+let xmlDecoderSuite = BenchmarkSuite(name: "XMLDecoder", settings: Iterations(10000), WarmupIterations(10)) { suite in
+    let xml = #"<test><a>hello</a><b><c>good</c><d>bye</d></b><b><c>good</c><d>bye</d></b><b><c>good</c><d>bye</d></b></test>"#
+    suite.benchmark("loadXML") {
+        let result = try XML.Element(xmlString: xml)
+    }
+
+    if let numbers = try? XML.Element(xmlString: #"<numbers><b>true</b><i>362222</i><f>3.14</f><d>2.777776</d></numbers>"#) {
+        suite.benchmark("numbers") {
+            _ = try XMLDecoder().decode(Numbers.self, from: numbers)
+        }
+    }
+
+    if let strings = try? XML.Element(xmlString: #"<strings><s>Benchmark testing XMLDecoder</s></strings>"#) {
+        suite.benchmark("strings") {
+            _ = try XMLDecoder().decode(Strings.self, from: strings)
+        }
+    }
+
+    if let arrays = try? XML.Element(xmlString: #"<strings><a1>23</a1><a1>89</a1><a1>28768234</a1><a2>test</a2></strings>"#) {
+        suite.benchmark("arrays") {
+            _ = try XMLDecoder().decode(Arrays.self, from: arrays)
+        }
+    }
+}
+
+/// Suite of benchmark tests for XMLEncoder
+let queryEncoderSuite = BenchmarkSuite(name: "QueryEncoder", settings: Iterations(10000), WarmupIterations(10)) { suite in
+    encoderSuite(for: QueryEncoder(), suite: suite)
+}

Benchmark/Sources/soto-benchmark/main.swift

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2020 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+
+let suites = [
+    awsSignerV4Suite,
+    queryEncoderSuite,
+    xmlEncoderSuite,
+    xmlDecoderSuite,
+    awsClientSuite,
+]
+
+Benchmark.main(suites)

scripts/sanity.sh

@@ -47,7 +47,7 @@ for language in swift-or-c; do
   matching_files=( -name '*' )
   case "$language" in
       swift-or-c)
-        exceptions=( -path '*Sources/INIParser/*' -o -path '*Sources/CSotoExpat/*' -o -name Package.swift)
+        exceptions=( -path '*Sources/INIParser/*' -o -path '*Sources/CSotoExpat/*' -o -path '*Benchmark/.build/*' -o -name Package.swift)
         matching_files=( -name '*.swift' -o -name '*.c' -o -name '*.h' )
         cat > "$tmp" <<"EOF"
 //===----------------------------------------------------------------------===//