From d7517a18e0e6922178925b9b57acd8ea19a27f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Mon, 11 Mar 2024 21:41:40 +0100 Subject: [PATCH 1/4] Retries establishing SSH connection --- .../SSH/VirtualMachineSSHClient.swift | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift index a88ac8f..eaa055f 100644 --- a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift +++ b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift @@ -5,6 +5,7 @@ import SSHDomain private enum VirtualMachineSSHClientError: LocalizedError, CustomDebugStringConvertible { case missingSSHUsername case missingSSHPassword + case failedConnectingToVirtualMachineAfterRetries var errorDescription: String? { debugDescription @@ -16,6 +17,8 @@ private enum VirtualMachineSSHClientError: LocalizedError, CustomDebugStringConv "The SSH username is not set in Tartelet's settings." case .missingSSHPassword: "The SSH password is not set in Tartelet's settings." + case .failedConnectingToVirtualMachineAfterRetries: + "Failed establishing connection to virtual machine after retrying a number of times." } } } @@ -41,11 +44,15 @@ public struct VirtualMachineSSHClient { self.connectionHandler = connectionHandler } - func connect(to virtualMachine: VirtualMachine) async throws -> SSHClientType.SSHConnectionType { + func connect(to virtualMachine: VirtualMachine) async throws { let ipAddress = try await getIPAddress(of: virtualMachine) - let connection = try await connectToVirtualMachine(named: virtualMachine.name, on: ipAddress) + let connection = try await connectToVirtualMachine( + named: virtualMachine.name, + on: ipAddress, + remainingAttempts: 3 + ) try await connectionHandler.didConnect(to: virtualMachine, through: connection) - return connection + try? await connection.close() } } @@ -62,6 +69,28 @@ private extension VirtualMachineSSHClient { } } + private func connectToVirtualMachine( + named virtualMachineName: String, + on host: String, + remainingAttempts: Int + ) async throws -> SSHClientType.SSHConnectionType { + do { + try Task.checkCancellation() + return try await connectToVirtualMachine(named: virtualMachineName, on: host) + } catch { + guard remainingAttempts > 1 else { + throw VirtualMachineSSHClientError.failedConnectingToVirtualMachineAfterRetries + } + try Task.checkCancellation() + try await Task.sleep(for: .seconds(2)) + return try await connectToVirtualMachine( + named: virtualMachineName, + on: host, + remainingAttempts: remainingAttempts - 1 + ) + } + } + private func connectToVirtualMachine( named virtualMachineName: String, on host: String From f2cd74948a7b906ccd6233066bff046fcbeda465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Mon, 11 Mar 2024 21:49:27 +0100 Subject: [PATCH 2/4] Restarts virtual machine when failing to establish SSH connection --- .../SSH/SSHConnectingVirtualMachine.swift | 104 +++++++++++++++++- Tartelet/Sources/Composers.swift | 1 + 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift index 77f1c1a..8242dfd 100644 --- a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift +++ b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift @@ -1,33 +1,95 @@ +import Foundation +import LoggingDomain import SSHDomain +private enum StartVirtualMachineValue { + case virtualMachineTerminated + case sshConnectionCompleted +} + +private enum StartVirtualMachineError: LocalizedError, CustomDebugStringConvertible { + case failedStartingVirtualMachine(Error) + case failedEstablishingSSHConnection(Error) + case cancelled + + var errorDescription: String? { + debugDescription + } + + var debugDescription: String { + switch self { + case .failedStartingVirtualMachine: + "Failed starting virtual machine" + case .failedEstablishingSSHConnection: + "Failed establishing SSH connection" + case .cancelled: + "Task was cancelled" + } + } +} + +private typealias StartVirtualMachineResult = Result + public final class SSHConnectingVirtualMachine: VirtualMachine { public var name: String { virtualMachine.name } + private let logger: Logger private let virtualMachine: VirtualMachine private let sshClient: VirtualMachineSSHClient public init( + logger: Logger, virtualMachine: VirtualMachine, sshClient: VirtualMachineSSHClient ) { + self.logger = logger self.virtualMachine = virtualMachine self.sshClient = sshClient } public func start() async throws { - let connectTask = Task { - let connection = try await self.sshClient.connect(to: self.virtualMachine) - try await connection.close() + try await withThrowingTaskGroup(of: StartVirtualMachineResult.self) { group in + group.addTask { + return try await self.startVirtualMachine() + } + group.addTask { + return try await self.connect(to: self.virtualMachine) + } + for try await result in group { + switch result { + case let .success(value): + switch value { + case .virtualMachineTerminated: + // In the happy path, the SSH connection has already been established, + // but we'll cancel it in the odd case that it hasn't. + group.cancelAll() + case .sshConnectionCompleted: + // Nothing to do. The virtual machine should keep running. + break + } + case let .failure(error): + switch error { + case .failedStartingVirtualMachine, .failedEstablishingSSHConnection: + // If we fail to start the virtual machine or establish the SSH connection, + // then we'll cancel the other operations. This ensures the virtual machine is + // shut down and enables the VirtualMachineFleet to start a new virtual machine. + group.cancelAll() + throw error + case .cancelled: + // The operation was canceled, so there's nothing left for us to do. + break + } + } + } } - try await self.virtualMachine.start() - connectTask.cancel() } public func clone(named newName: String) async throws -> VirtualMachine { let virtualMachine = try await virtualMachine.clone(named: newName) return SSHConnectingVirtualMachine( + logger: logger, virtualMachine: virtualMachine, sshClient: sshClient ) @@ -41,3 +103,35 @@ public final class SSHConnectingVirtualMachine: Virtua try await virtualMachine.getIPAddress() } } + +private extension SSHConnectingVirtualMachine { + private func startVirtualMachine() async throws -> StartVirtualMachineResult { + do { + try await self.virtualMachine.start() + return .success(.virtualMachineTerminated) + } catch { + if error is CancellationError { + return .failure(.cancelled) + } else { + return .failure(.failedStartingVirtualMachine(error)) + } + } + } + + private func connect(to virtualMachine: VirtualMachine) async throws -> StartVirtualMachineResult { + do { + try await sshClient.connect(to: virtualMachine) + return .success(.sshConnectionCompleted) + } catch { + if error is CancellationError { + return .failure(.cancelled) + } else { + logger.error(error.localizedDescription) + logger.error( + "Could not connect to the virtual machine over SSH, so the virtual machine will be shut down." + ) + return .failure(.failedEstablishingSSHConnection(error)) + } + } + } +} diff --git a/Tartelet/Sources/Composers.swift b/Tartelet/Sources/Composers.swift index 6cc926b..faa49ff 100644 --- a/Tartelet/Sources/Composers.swift +++ b/Tartelet/Sources/Composers.swift @@ -18,6 +18,7 @@ enum Composers { static let fleet = VirtualMachineFleet( logger: logger(subsystem: "VirtualMachineFleet"), baseVirtualMachine: SSHConnectingVirtualMachine( + logger: logger(subsystem: "SSHConnectingVirtualMachine"), virtualMachine: SettingsVirtualMachine( tart: Tart( homeProvider: SettingsTartHomeProvider( From 512a06d0d7c50f8f882a6867b9016fc08cf43c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Mon, 11 Mar 2024 21:51:52 +0100 Subject: [PATCH 3/4] Fixes compile issue --- .../VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift index eaa055f..96211fc 100644 --- a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift +++ b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/VirtualMachineSSHClient.swift @@ -44,7 +44,7 @@ public struct VirtualMachineSSHClient { self.connectionHandler = connectionHandler } - func connect(to virtualMachine: VirtualMachine) async throws { + func connect(to virtualMachine: VirtualMachine) async throws -> SSHClientType.SSHConnectionType { let ipAddress = try await getIPAddress(of: virtualMachine) let connection = try await connectToVirtualMachine( named: virtualMachine.name, @@ -52,7 +52,7 @@ public struct VirtualMachineSSHClient { remainingAttempts: 3 ) try await connectionHandler.didConnect(to: virtualMachine, through: connection) - try? await connection.close() + return connection } } From 417e2e864cf7bcc3e0a3d0c2e0eed21172e17ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=B8vring?= Date: Mon, 11 Mar 2024 21:56:32 +0100 Subject: [PATCH 4/4] Closes SSH connection --- .../VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift index 8242dfd..b417db6 100644 --- a/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift +++ b/Packages/VirtualMachine/Sources/VirtualMachineDomain/SSH/SSHConnectingVirtualMachine.swift @@ -120,7 +120,8 @@ private extension SSHConnectingVirtualMachine { private func connect(to virtualMachine: VirtualMachine) async throws -> StartVirtualMachineResult { do { - try await sshClient.connect(to: virtualMachine) + let connection = try await sshClient.connect(to: virtualMachine) + try await connection.close() return .success(.sshConnectionCompleted) } catch { if error is CancellationError {