diff --git a/Sources/SotoCore/AWSClient.swift b/Sources/SotoCore/AWSClient.swift index 50f8a9ac7..8a97877f5 100644 --- a/Sources/SotoCore/AWSClient.swift +++ b/Sources/SotoCore/AWSClient.swift @@ -48,7 +48,7 @@ public final class AWSClient: Sendable { /// HTTP client used by AWSClient public let httpClient: AWSHTTPClient /// Logger used for non-request based output - let clientLogger: Logger + public let logger: Logger /// client options let options: Options @@ -71,13 +71,13 @@ public final class AWSClient: Sendable { middleware: some AWSMiddlewareProtocol, options: Options = Options(), httpClient: AWSHTTPClient = HTTPClient.shared, - logger clientLogger: Logger = AWSClient.loggingDisabled + logger: Logger = AWSClient.loggingDisabled ) { self.httpClient = httpClient let credentialProvider = credentialProviderFactory.createProvider( context: .init( httpClient: self.httpClient, - logger: clientLogger, + logger: logger, options: options ) ) @@ -88,7 +88,7 @@ public final class AWSClient: Sendable { RetryMiddleware(retryPolicy: retryPolicyFactory.retryPolicy) ErrorHandlingMiddleware(options: options) } - self.clientLogger = clientLogger + self.logger = logger self.options = options } @@ -105,13 +105,13 @@ public final class AWSClient: Sendable { retryPolicy retryPolicyFactory: RetryPolicyFactory = .default, options: Options = Options(), httpClient: AWSHTTPClient = HTTPClient.shared, - logger clientLogger: Logger = AWSClient.loggingDisabled + logger: Logger = AWSClient.loggingDisabled ) { self.httpClient = httpClient let credentialProvider = credentialProviderFactory.createProvider( context: .init( httpClient: self.httpClient, - logger: clientLogger, + logger: logger, options: options ) ) @@ -121,7 +121,7 @@ public final class AWSClient: Sendable { RetryMiddleware(retryPolicy: retryPolicyFactory.retryPolicy) ErrorHandlingMiddleware(options: options) } - self.clientLogger = clientLogger + self.logger = logger self.options = options } @@ -402,9 +402,15 @@ extension AWSClient { try Task.checkCancellation() // combine service and client middleware stacks let middlewareStack = config.middleware.map { AWSDynamicMiddlewareStack($0, self.middleware) } ?? self.middleware + let credential = try await self.credentialProvider.getCredential(logger: logger) let middlewareContext = AWSMiddlewareContext( operation: operationName, serviceConfig: config, + credential: StaticCredential( + accessKeyId: credential.accessKeyId, + secretAccessKey: credential.secretAccessKey, + sessionToken: credential.sessionToken + ), logger: logger ) // run middleware stack with httpClient execute at the end diff --git a/Sources/SotoCore/Credential/Credential+IsEmpty.swift b/Sources/SotoCore/Credential/Credential+IsEmpty.swift index c26f11087..d4fa68b75 100644 --- a/Sources/SotoCore/Credential/Credential+IsEmpty.swift +++ b/Sources/SotoCore/Credential/Credential+IsEmpty.swift @@ -18,4 +18,8 @@ extension Credential { func isEmpty() -> Bool { self.accessKeyId.isEmpty || self.secretAccessKey.isEmpty } + + func getStaticCredential() -> StaticCredential { + .init(accessKeyId: accessKeyId, secretAccessKey: secretAccessKey, sessionToken: sessionToken) + } } diff --git a/Sources/SotoCore/Credential/CredentialProvider.swift b/Sources/SotoCore/Credential/CredentialProvider.swift index 03e13ee02..a9a9b94e1 100644 --- a/Sources/SotoCore/Credential/CredentialProvider.swift +++ b/Sources/SotoCore/Credential/CredentialProvider.swift @@ -170,7 +170,7 @@ extension CredentialProviderFactory { /// Don't supply any credentials public static var empty: CredentialProviderFactory { Self { _ in - StaticCredential(accessKeyId: "", secretAccessKey: "") + EmptyCredential() } } diff --git a/Sources/SotoCore/Credential/EmptyCredential.swift b/Sources/SotoCore/Credential/EmptyCredential.swift new file mode 100644 index 000000000..d2b562499 --- /dev/null +++ b/Sources/SotoCore/Credential/EmptyCredential.swift @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Soto for AWS open source project +// +// Copyright (c) 2017-2023 the Soto project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Soto project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/// Empty credentials +public struct EmptyCredential: CredentialProvider, Credential { + public var accessKeyId: String { "" } + public var secretAccessKey: String { "" } + public var sessionToken: String? { nil } + + public func getCredential(logger: Logger) async throws -> any Credential { + self + } +} diff --git a/Sources/SotoCore/HTTP/AWSHTTPBody.swift b/Sources/SotoCore/HTTP/AWSHTTPBody.swift index 2418a3b69..0326f484f 100644 --- a/Sources/SotoCore/HTTP/AWSHTTPBody.swift +++ b/Sources/SotoCore/HTTP/AWSHTTPBody.swift @@ -74,6 +74,15 @@ public struct AWSHTTPBody: Sendable { } } + public var isEmpty: Bool { + switch self.storage { + case .byteBuffer(let buffer): + return buffer.readableBytes == 0 + case .asyncSequence(_, let length): + return length == 0 + } + } + public var isStreaming: Bool { switch self.storage { case .byteBuffer: diff --git a/Sources/SotoCore/Middleware/Middleware.swift b/Sources/SotoCore/Middleware/Middleware.swift index 036ae7054..fdaaefa9b 100644 --- a/Sources/SotoCore/Middleware/Middleware.swift +++ b/Sources/SotoCore/Middleware/Middleware.swift @@ -18,6 +18,7 @@ import Logging public struct AWSMiddlewareContext: Sendable { public var operation: String public var serviceConfig: AWSServiceConfig + public var credential: StaticCredential public var logger: Logger } diff --git a/Sources/SotoCore/Middleware/Middleware/S3Middleware.swift b/Sources/SotoCore/Middleware/Middleware/S3Middleware.swift index aa93e88ed..256b20b68 100644 --- a/Sources/SotoCore/Middleware/Middleware/S3Middleware.swift +++ b/Sources/SotoCore/Middleware/Middleware/S3Middleware.swift @@ -22,6 +22,18 @@ internal import SotoXML @_implementationOnly import SotoXML #endif +extension S3Middleware { + @TaskLocal public static var executionContext: ExecutionContext? + + /// Task local execution context for operation + public struct ExecutionContext: Sendable { + public init(useS3ExpressControlEndpoint: Bool) { + self.useS3ExpressControlEndpoint = useS3ExpressControlEndpoint + } + public let useS3ExpressControlEndpoint: Bool + } +} + /// Middleware applied to S3 service /// /// This middleware does a number of request and response fixups for the S3 service. @@ -68,62 +80,60 @@ public struct S3Middleware: AWSMiddlewareProtocol { if request.url.path.hasPrefix("/arn:") { return try await handleARNBucket(request, context: context, next: next) } - /// process URL into form ${bucket}.s3.amazon.com + /// split path into components. Drop first as it is an empty string let paths = request.url.path.split(separator: "/", maxSplits: 2, omittingEmptySubsequences: false).dropFirst() guard let bucket = paths.first, var host = request.url.host else { return try await next(request, context) } + let path = paths.dropFirst().first.flatMap { String($0) } ?? "" if let port = request.url.port { host = "\(host):\(port)" } var urlPath: String var urlHost: String - let isAmazonUrl = host.hasSuffix("amazonaws.com") + let isAmazonUrl = host.hasSuffix(context.serviceConfig.region.partition.dnsSuffix) - var hostComponents = host.split(separator: ".") - if isAmazonUrl, context.serviceConfig.options.contains(.s3UseTransferAcceleratedEndpoint) { - if let s3Index = hostComponents.firstIndex(where: { $0 == "s3" }) { - var s3 = "s3" - s3 += "-accelerate" - // assume next host component is region - let regionIndex = s3Index + 1 - hostComponents.remove(at: regionIndex) - hostComponents[s3Index] = Substring(s3) - host = hostComponents.joined(separator: ".") - } - } + // if bucket has suffix "-x-s3" then it must be an s3 express directory bucket and the endpoint needs set up + // to use s3express + if bucket.hasSuffix("--x-s3"), + let response = try await handleS3ExpressBucketEndpoint( + request: request, + bucket: bucket, + path: path, + context: context, + next: next + ) + { + return response + } else { - // Is bucket an ARN - if bucket.hasPrefix("arn:") { - guard let arn = ARN(string: bucket), - let resourceType = arn.resourceType, - let region = arn.region, - let accountId = arn.accountId - else { - throw AWSClient.ClientError.invalidARN - } - guard resourceType == "accesspoint", arn.service == "s3-object-lambda" || arn.service == "s3-outposts" else { - throw AWSClient.ClientError.invalidARN + // Should we use accelerated endpoint + var hostComponents = host.split(separator: ".") + if isAmazonUrl, context.serviceConfig.options.contains(.s3UseTransferAcceleratedEndpoint) { + if let s3Index = hostComponents.firstIndex(where: { $0 == "s3" }) { + var s3 = "s3" + s3 += "-accelerate" + // assume next host component is region + let regionIndex = s3Index + 1 + hostComponents.remove(at: regionIndex) + hostComponents[s3Index] = Substring(s3) + host = hostComponents.joined(separator: ".") + } } - urlPath = "/" - // https://tutorial-object-lambda-accesspoint-123456789012.s3-object-lambda.us-west-2.amazonaws.com:443 - urlHost = "\(arn.resourceId)-\(resourceType)-\(accountId).\(arn.service).\(region).amazonaws.com" - // if host name contains amazonaws.com and bucket name doesn't contain a period do virtual address look up - } else if isAmazonUrl || context.serviceConfig.options.contains(.s3ForceVirtualHost), !bucket.contains(".") { - let pathsWithoutBucket = paths.dropFirst() // bucket - urlPath = pathsWithoutBucket.first.flatMap { String($0) } ?? "" - - if hostComponents.first == bucket { - // Bucket name is part of host. No need to append bucket - urlHost = host + if isAmazonUrl || context.serviceConfig.options.contains(.s3ForceVirtualHost), !bucket.contains(".") { + urlPath = path + if hostComponents.first == bucket { + // Bucket name is part of host. No need to append bucket + urlHost = host + } else { + urlHost = "\(bucket).\(host)" + } } else { - urlHost = "\(bucket).\(host)" + urlPath = paths.joined(separator: "/") + urlHost = host } - } else { - urlPath = paths.joined(separator: "/") - urlHost = host } - let request = Self.updateRequestURL(request, host: urlHost, path: urlPath) + let request = try Self.updateRequestURL(request, host: urlHost, path: urlPath) return try await next(request, context) } @@ -155,8 +165,8 @@ public struct S3Middleware: AWSMiddlewareProtocol { let path = String(resourceIDSplit.dropFirst().first ?? "") let service = String(arn.service) let serviceIdentifier = service != "s3" ? service : "s3-accesspoint" - let urlHost = "\(bucket)-\(accountId).\(serviceIdentifier).\(region).amazonaws.com" - let request = Self.updateRequestURL(request, host: urlHost, path: path) + let urlHost = "\(bucket)-\(accountId).\(serviceIdentifier).\(region).\(context.serviceConfig.region.partition.dnsSuffix)" + let request = try Self.updateRequestURL(request, host: urlHost, path: path) var context = context context.serviceConfig = AWSServiceConfig( @@ -177,16 +187,62 @@ public struct S3Middleware: AWSMiddlewareProtocol { return try await next(request, context) } + /// Handle bucket names in the form of an ARN + /// - Parameters: + /// - request: request + /// - context: request context + /// - next: next handler + /// - Returns: returns response from next handler + func handleS3ExpressBucketEndpoint( + request: AWSHTTPRequest, + bucket: Substring, + path: String, + context: AWSMiddlewareContext, + next: AWSMiddlewareNextHandler + ) async throws -> AWSHTTPResponse? { + // Note this uses my own version of split (as the Swift one requires macOS 13) + let split = bucket.split(separator: "--") + guard split.count > 2, split.last == "x-s3" else { return nil } + let zone = split[split.count - 2] + let urlHost: String + // should we use a control endpoint or a zone endpoint + if let executionContext = Self.executionContext, + executionContext.useS3ExpressControlEndpoint + { + urlHost = "s3express-control.\(context.serviceConfig.region).\(context.serviceConfig.region.partition.dnsSuffix)/\(bucket)" + } else { + urlHost = "\(bucket).s3express-\(zone).\(context.serviceConfig.region).\(context.serviceConfig.region.partition.dnsSuffix)" + } + let request = try Self.updateRequestURL(request, host: urlHost, path: path) + var context = context + context.serviceConfig = AWSServiceConfig( + region: context.serviceConfig.region, + partition: context.serviceConfig.region.partition, + serviceName: "S3", + serviceIdentifier: "s3", + signingName: "s3express", + serviceProtocol: context.serviceConfig.serviceProtocol, + apiVersion: context.serviceConfig.apiVersion, + errorType: context.serviceConfig.errorType, + xmlNamespace: context.serviceConfig.xmlNamespace, + middleware: context.serviceConfig.middleware, + timeout: context.serviceConfig.timeout, + byteBufferAllocator: context.serviceConfig.byteBufferAllocator, + options: context.serviceConfig.options + ) + return try await next(request, context) + } + /// Update request with new host and path /// - Parameters: /// - request: request /// - host: new host name - /// - path: new path + /// - path: new path (without forward slash prefix) /// - Returns: new request - static func updateRequestURL(_ request: AWSHTTPRequest, host: some StringProtocol, path: String) -> AWSHTTPRequest { + static func updateRequestURL(_ request: AWSHTTPRequest, host: some StringProtocol, path: String) throws -> AWSHTTPRequest { var path = path // add trailing "/" back if it was present, no need to check for single slash path here - if request.url.pathWithSlash.hasSuffix("/") { + if request.url.pathWithSlash.hasSuffix("/"), path.count > 0 { path += "/" } // add percent encoding back into path as converting from URL to String has removed it @@ -196,7 +252,10 @@ public struct S3Middleware: AWSMiddlewareProtocol { urlString += "?\(query)" } var request = request - request.url = URL(string: urlString)! + guard let url = URL(string: urlString) else { + throw AWSClient.ClientError.invalidURL + } + request.url = url return request } @@ -210,16 +269,18 @@ public struct S3Middleware: AWSMiddlewareProtocol { switch context.operation { // fixup CreateBucket to include location case "CreateBucket": - var xml = "" - if context.serviceConfig.region != .useast1 { - xml += "" - xml += "" - xml += context.serviceConfig.region.rawValue - xml += "" - xml += "" + if request.body.isEmpty { + var xml = "" + if context.serviceConfig.region != .useast1 { + xml += "" + xml += "" + xml += context.serviceConfig.region.rawValue + xml += "" + xml += "" + } + // TODO: pass service config down so we can use the ByteBufferAllocator + request.body = .init(string: xml) } - // TODO: pass service config down so we can use the ByteBufferAllocator - request.body = .init(string: xml) default: break @@ -263,3 +324,35 @@ public struct S3Middleware: AWSMiddlewareProtocol { return error } } + +extension StringProtocol { + func split(separator: some StringProtocol) -> [Self.SubSequence] { + var split: [Self.SubSequence] = [] + var index: Self.Index = self.startIndex + var startSplit: Self.Index = self.startIndex + while index != self.endIndex { + if let end = self[index...].prefixEnd(separator) { + split.append(self[startSplit.. Self.Index? { + var prefixIndex = prefix.startIndex + var index = self.startIndex + while prefixIndex != prefix.endIndex { + if self[index] != prefix[prefixIndex] { + return nil + } + index = self.index(after: index) + prefixIndex = prefix.index(after: prefixIndex) + } + return index + } +} diff --git a/Sources/SotoCore/Middleware/Middleware/SigningMiddleware.swift b/Sources/SotoCore/Middleware/Middleware/SigningMiddleware.swift index aafefa81e..ebde0e306 100644 --- a/Sources/SotoCore/Middleware/Middleware/SigningMiddleware.swift +++ b/Sources/SotoCore/Middleware/Middleware/SigningMiddleware.swift @@ -24,10 +24,12 @@ struct SigningMiddleware: AWSMiddlewareProtocol { @inlinable func handle(_ request: AWSHTTPRequest, context: AWSMiddlewareContext, next: AWSMiddlewareNextHandler) async throws -> AWSHTTPResponse { var request = request - // get credentials - let credential = try await self.credentialProvider.getCredential(logger: context.logger) // construct signer - let signer = AWSSigner(credentials: credential, name: context.serviceConfig.signingName, region: context.serviceConfig.region.rawValue) + let signer = AWSSigner( + credentials: context.credential, + name: context.serviceConfig.signingName, + region: context.serviceConfig.region.rawValue + ) request.signHeaders(signer: signer, serviceConfig: context.serviceConfig) return try await next(request, context) } diff --git a/Tests/SotoCoreTests/AWSServiceTests.swift b/Tests/SotoCoreTests/AWSServiceTests.swift index cf13ce32f..40a3a5288 100644 --- a/Tests/SotoCoreTests/AWSServiceTests.swift +++ b/Tests/SotoCoreTests/AWSServiceTests.swift @@ -129,7 +129,12 @@ class AWSServiceTests: XCTestCase { let service = TestService(client: client, config: createServiceConfig()) let service2 = service.with(middleware: TestMiddleware()) let request = AWSHTTPRequest(url: URL(string: "http://testurl.com")!, method: .GET, headers: [:], body: .init()) - let context = AWSMiddlewareContext(operation: "TestURL", serviceConfig: service2.config, logger: TestEnvironment.logger) + let context = AWSMiddlewareContext( + operation: "TestURL", + serviceConfig: service2.config, + credential: EmptyCredential().getStaticCredential(), + logger: TestEnvironment.logger + ) let response = try await service2.config.middleware!.handle(request, context: context) { request, _ in .init(status: .ok, headers: request.headers, body: request.body) } diff --git a/Tests/SotoCoreTests/Credential/RuntimeSelectorCredentialProviderTests.swift b/Tests/SotoCoreTests/Credential/RuntimeSelectorCredentialProviderTests.swift index 986ded0ed..1b16ecdac 100644 --- a/Tests/SotoCoreTests/Credential/RuntimeSelectorCredentialProviderTests.swift +++ b/Tests/SotoCoreTests/Credential/RuntimeSelectorCredentialProviderTests.swift @@ -81,7 +81,7 @@ class RuntimeSelectorCredentialProviderTests: XCTestCase { XCTAssertEqual(credential.secretAccessKey, "") XCTAssertEqual(credential.sessionToken, nil) let internalProvider = try await (client.credentialProvider as? RuntimeSelectorCredentialProvider)?.getCredentialProviderTask() - XCTAssert(internalProvider is StaticCredential) + XCTAssert(internalProvider is EmptyCredential) } func testFoundSelectorWithOneProvider() async throws { @@ -90,7 +90,7 @@ class RuntimeSelectorCredentialProviderTests: XCTestCase { defer { XCTAssertNoThrow(try client.syncShutdown()) } let credential = try await client.credentialProvider.getCredential(logger: TestEnvironment.logger) XCTAssert(credential.isEmpty()) - XCTAssert(client.credentialProvider is StaticCredential) + XCTAssert(client.credentialProvider is EmptyCredential) } func testECSProvider() async throws { @@ -188,7 +188,6 @@ class RuntimeSelectorCredentialProviderTests: XCTestCase { defer { XCTAssertNoThrow(try client.syncShutdown()) } _ = try await client.credentialProvider.getCredential(logger: TestEnvironment.logger) let internalProvider = try await (client.credentialProvider as? RuntimeSelectorCredentialProvider)?.getCredentialProviderTask() - XCTAssert(internalProvider is StaticCredential) - XCTAssert((internalProvider as? StaticCredential)?.isEmpty() == true) + XCTAssert(internalProvider is EmptyCredential) } } diff --git a/Tests/SotoCoreTests/MiddlewareTests.swift b/Tests/SotoCoreTests/MiddlewareTests.swift index 869107689..4242cd2a0 100644 --- a/Tests/SotoCoreTests/MiddlewareTests.swift +++ b/Tests/SotoCoreTests/MiddlewareTests.swift @@ -43,7 +43,7 @@ class MiddlewareTests: XCTestCase { let client = createAWSClient(credentialProvider: .empty) let config = createServiceConfig( region: .useast1, - endpoint: "https://\(serviceName).us-east-1.amazonaws.com", + service: serviceName, middlewares: AWSMiddlewareStack { middleware CatchRequestMiddleware() @@ -218,6 +218,33 @@ class MiddlewareTests: XCTestCase { } } + func testS3MiddlewareS3ExpressEndpoint() async throws { + // Test virual address + try await self.testMiddleware( + S3Middleware(), + serviceName: "s3", + uri: "/s3express--bucket--use1-az6--x-s3/file" + ) { request, _ in + XCTAssertEqual(request.url.absoluteString, "https://s3express--bucket--use1-az6--x-s3.s3express-use1-az6.us-east-1.amazonaws.com/file") + } + } + + func testS3MiddlewareS3ExpressControlEndpoint() async throws { + // Test virual address + try await S3Middleware.$executionContext.withValue(.init(useS3ExpressControlEndpoint: true)) { + try await self.testMiddleware( + S3Middleware(), + serviceName: "s3", + uri: "/s3express--bucket--use1-az6--x-s3/" + ) { request, _ in + XCTAssertEqual( + request.url.absoluteString, + "https://s3express-control.us-east-1.amazonaws.com/s3express--bucket--use1-az6--x-s3/" + ) + } + } + } + // create a buffer of random values. Will always create the same given you supply the same z and w values // Random number generator from https://www.codeproject.com/Articles/25172/Simple-Random-Number-Generation func createRandomBuffer(_ w: UInt, _ z: UInt, size: Int) -> [UInt8] {