Migrate to Rust backend with fast spendability support

Package.resolved

@@ -104,7 +104,7 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/zcash-hackworks/zcash-light-client-ffi",
       "state" : {
-        "revision" : "cdbc06f10b2d7cbe0d6362b30f68167825942e86"
+        "revision" : "57eb3bd4db3c26bf44d2d8d8b0d6f09f7602a125"
       }
     }
   ],

Package.swift

@@ -16,7 +16,7 @@ let package = Package(
     dependencies: [
         .package(url: "https://github.com/grpc/grpc-swift.git", from: "1.14.0"),
         .package(url: "https://github.com/stephencelis/SQLite.swift.git", from: "0.14.1"),
-        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", revision: "cdbc06f10b2d7cbe0d6362b30f68167825942e86")
+        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", revision: "57eb3bd4db3c26bf44d2d8d8b0d6f09f7602a125")
     ],
     targets: [
         .target(

Sources/ZcashLightClientKit/Block/Actions/Action.swift

@@ -36,7 +36,6 @@ enum CBPState: CaseIterable {
     case validateServer
     case computeSyncControlData
     case download
-    case validate
     case scan
     case clearAlreadyScannedBlocks
     case enhance

Sources/ZcashLightClientKit/Block/Actions/DownloadAction.swift

@@ -21,7 +21,7 @@ final class DownloadAction {
     }
 
     private func update(context: ActionContext) async -> ActionContext {
-        await context.update(state: .validate)
+        await context.update(state: .scan)
         return context
     }
 }

Sources/ZcashLightClientKit/Block/CompactBlockProcessor.swift

@@ -218,8 +218,6 @@ actor CompactBlockProcessor {
                 action = ComputeSyncControlDataAction(container: container, configProvider: configProvider)
             case .download:
                 action = DownloadAction(container: container, configProvider: configProvider)
-            case .validate:
-                action = ValidateAction(container: container)
             case .scan:
                 action = ScanAction(container: container, configProvider: configProvider)
             case .clearAlreadyScannedBlocks:
@@ -591,8 +589,6 @@ extension CompactBlockProcessor {
             break
         case .download:
             break
-        case .validate:
-            break
         case .scan:
             break
         case .clearAlreadyScannedBlocks:

Sources/ZcashLightClientKit/Block/Scan/BlockScanner.swift

@@ -50,13 +50,14 @@ extension BlockScannerImpl: BlockScanner {
             try Task.checkCancellation()
 
             let previousScannedHeight = lastScannedHeight
+            let startHeight = previousScannedHeight + 1
 
             // TODO: [#576] remove this arbitrary batch size https://github.com/zcash/ZcashLightClientKit/issues/576
-            let batchSize = scanBatchSize(startScanHeight: previousScannedHeight + 1, network: config.networkType)
+            let batchSize = scanBatchSize(startScanHeight: startHeight, network: config.networkType)
 
             let scanStartTime = Date()
             do {
-                try await self.rustBackend.scanBlocks(limit: batchSize)
+                try await self.rustBackend.scanBlocks(fromHeight: Int32(startHeight), limit: batchSize)
             } catch {
                 logger.debug("block scanning failed with error: \(String(describing: error))")
                 throw error

Sources/ZcashLightClientKit/Entity/SubtreeRoot.swift

@@ -0,0 +1,13 @@
+//
+//  SubtreeRoot.swift
+//
+//
+//  Created by Jack Grigg on 19/07/2023.
+//
+
+import Foundation
+
+public struct SubtreeRoot {
+    public let rootHash: Data
+    public let completingBlockHeight: BlockHeight
+}

Sources/ZcashLightClientKit/Error/ZcashError.swift

@@ -276,6 +276,21 @@ public enum ZcashError: Equatable, Error {
     /// Transparent receiver generated by rust layer is invalid when calling ZcashRustBackend.getTransparentReceiver
     /// ZRUST0045
     case rustGetTransparentReceiverInvalidReceiver
+    /// Unable to allocate memory required to write blocks when calling ZcashRustBackend.putSaplingSubtreeRoots
+    /// ZRUST0046
+    case rustPutSaplingSubtreeRootsAllocationProblem
+    /// Error from rust layer when calling ZcashRustBackend.putSaplingSubtreeRoots
+    /// - `rustError` contains error generated by the rust layer.
+    /// ZRUST0047
+    case rustPutSaplingSubtreeRoots(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.updateChainTip
+    /// - `rustError` contains error generated by the rust layer.
+    /// ZRUST0048
+    case rustUpdateChainTip(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.suggestScanRanges
+    /// - `rustError` contains error generated by the rust layer.
+    /// ZRUST0049
+    case rustSuggestScanRanges(_ rustError: String)
     /// SQLite query failed when fetching all accounts from the database.
     /// - `sqliteError` is error produced by SQLite library.
     /// ZADAO0001
@@ -605,6 +620,10 @@ public enum ZcashError: Equatable, Error {
         case .rustGetSaplingReceiverInvalidReceiver: return "Sapling receiver generated by rust layer is invalid when calling ZcashRustBackend.getSaplingReceiver"
         case .rustGetTransparentReceiverInvalidAddress: return "Error from rust layer when calling ZcashRustBackend.getTransparentReceiver"
         case .rustGetTransparentReceiverInvalidReceiver: return "Transparent receiver generated by rust layer is invalid when calling ZcashRustBackend.getTransparentReceiver"
+        case .rustPutSaplingSubtreeRootsAllocationProblem: return "Unable to allocate memory required to store subtree roots when calling ZcashRustBackend.putSaplingSubtreeRoots"
+        case .rustPutSaplingSubtreeRoots: return "Error from rust layer when calling ZcashRustBackend.putSaplingSubtreeRoots"
+        case .rustUpdateChainTip: return "Error from rust layer when calling ZcashRustBackend.updateChainTip"
+        case .rustSuggestScanRanges: return "Error from rust layer when calling ZcashRustBackend.suggestScanRanges"
         case .accountDAOGetAll: return "SQLite query failed when fetching all accounts from the database."
         case .accountDAOGetAllCantDecode: return "Fetched accounts from SQLite but can't decode them."
         case .accountDAOFindBy: return "SQLite query failed when seaching for accounts in the database."
@@ -763,6 +782,10 @@ public enum ZcashError: Equatable, Error {
         case .rustGetSaplingReceiverInvalidReceiver: return .rustGetSaplingReceiverInvalidReceiver
         case .rustGetTransparentReceiverInvalidAddress: return .rustGetTransparentReceiverInvalidAddress
         case .rustGetTransparentReceiverInvalidReceiver: return .rustGetTransparentReceiverInvalidReceiver
+        case .rustPutSaplingSubtreeRootsAllocationProblem: return .rustPutSaplingSubtreeRootsAllocationProblem
+        case .rustPutSaplingSubtreeRoots: return .rustPutSaplingSubtreeRoots
+        case .rustUpdateChainTip: return .rustUpdateChainTip
+        case .rustSuggestScanRanges: return .rustSuggestScanRanges
         case .accountDAOGetAll: return .accountDAOGetAll
         case .accountDAOGetAllCantDecode: return .accountDAOGetAllCantDecode
         case .accountDAOFindBy: return .accountDAOFindBy

Sources/ZcashLightClientKit/Error/ZcashErrorCode.swift

@@ -153,6 +153,14 @@ public enum ZcashErrorCode: String {
     case rustGetTransparentReceiverInvalidAddress = "ZRUST0044"
     /// Transparent receiver generated by rust layer is invalid when calling ZcashRustBackend.getTransparentReceiver
     case rustGetTransparentReceiverInvalidReceiver = "ZRUST0045"
+    /// Unable to allocate memory required to write blocks when calling ZcashRustBackend.putSaplingSubtreeRoots
+    case rustPutSaplingSubtreeRootsAllocationProblem = "ZRUST0046"
+    /// Error from rust layer when calling ZcashRustBackend.putSaplingSubtreeRoots
+    case rustPutSaplingSubtreeRoots = "ZRUST0047"
+    /// Error from rust layer when calling ZcashRustBackend.updateChainTip
+    case rustUpdateChainTip = "ZRUST0048"
+    /// Error from rust layer when calling ZcashRustBackend.suggestScanRanges
+    case rustSuggestScanRanges = "ZRUST0049"
     /// SQLite query failed when fetching all accounts from the database.
     case accountDAOGetAll = "ZADAO0001"
     /// Fetched accounts from SQLite but can't decode them.

Sources/ZcashLightClientKit/Model/ScanRange.swift

@@ -0,0 +1,13 @@
+//
+//  ScanRange.swift
+//
+//
+//  Created by Jack Grigg on 17/07/2023.
+//
+
+import Foundation
+
+struct ScanRange {
+    let range: Range<BlockHeight>
+    let priority: UInt8
+}

Sources/ZcashLightClientKit/Rust/ZcashRustBackend.swift

@@ -490,21 +490,6 @@ actor ZcashRustBackend: ZcashRustBackendWelding {
         }
     }
 
-    func validateCombinedChain(limit: UInt32 = 0) async throws {
-        let result = zcashlc_validate_combined_chain(fsBlockDbRoot.0, fsBlockDbRoot.1, dbData.0, dbData.1, limit, networkType.networkId)
-
-        switch result {
-        case -1:
-            return
-        case 0:
-            throw ZcashError.rustValidateCombinedChainValidationFailed(
-                lastErrorMessage(fallback: "`validateCombinedChain` failed with unknown error")
-            )
-        default:
-            throw ZcashError.rustValidateCombinedChainInvalidChain(result)
-        }
-    }
-
     func rewindToHeight(height: Int32) async throws {
         let result = zcashlc_rewind_to_height(dbData.0, dbData.1, height, networkType.networkId)
 
@@ -521,8 +506,97 @@ actor ZcashRustBackend: ZcashRustBackendWelding {
         }
     }
 
-    func scanBlocks(limit: UInt32 = 0) async throws {
-        let result = zcashlc_scan_blocks(fsBlockDbRoot.0, fsBlockDbRoot.1, dbData.0, dbData.1, limit, networkType.networkId)
+    func putSaplingSubtreeRoots(startIndex: UInt64, roots: [SubtreeRoot]) async throws {
+        var ffiSubtreeRootsVec: [FfiSubtreeRoot] = []
+
+        for root in roots {
+            let hashPtr = UnsafeMutablePointer<UInt8>.allocate(capacity: root.rootHash.count)
+
+            let contiguousHashBytes = ContiguousArray(root.rootHash.bytes)
+
+            let result: Void? = contiguousHashBytes.withContiguousStorageIfAvailable { hashBytesPtr in
+                // swiftlint:disable:next force_unwrapping
+                hashPtr.initialize(from: hashBytesPtr.baseAddress!, count: hashBytesPtr.count)
+            }
+
+            guard result != nil else {
+                defer {
+                    hashPtr.deallocate()
+                    ffiSubtreeRootsVec.deallocateElements()
+                }
+                throw ZcashError.rustPutSaplingSubtreeRootsAllocationProblem
+            }
+
+            ffiSubtreeRootsVec.append(
+                FfiSubtreeRoot(
+                    root_hash_ptr: hashPtr,
+                    root_hash_ptr_len: UInt(contiguousHashBytes.count),
+                    completing_block_height: UInt32(root.completingBlockHeight)
+                )
+            )
+        }
+
+        var contiguousFfiRoots = ContiguousArray(ffiSubtreeRootsVec)
+
+        let len = UInt(contiguousFfiRoots.count)
+
+        let rootsPtr = UnsafeMutablePointer<FfiSubtreeRoots>.allocate(capacity: 1)
+
+        defer { ffiSubtreeRootsVec.deallocateElements() }
+
+        try contiguousFfiRoots.withContiguousMutableStorageIfAvailable { ptr in
+            var roots = FfiSubtreeRoots()
+            roots.ptr = ptr.baseAddress
+            roots.len = len
+
+            rootsPtr.initialize(to: roots)
+
+            let res = zcashlc_put_sapling_subtree_roots(dbData.0, dbData.1, startIndex, rootsPtr, networkType.networkId)
+
+            guard res else {
+                throw ZcashError.rustPutSaplingSubtreeRoots(lastErrorMessage(fallback: "`putSaplingSubtreeRoots` failed with unknown error"))
+            }
+        }
+    }
+
+    func updateChainTip(height: Int32) async throws {
+        let result = zcashlc_update_chain_tip(dbData.0, dbData.1, height, networkType.networkId)
+
+        guard result else {
+            throw ZcashError.rustUpdateChainTip(lastErrorMessage(fallback: "`updateChainTip` failed with unknown error"))
+        }
+    }
+
+    func suggestScanRanges() async throws -> [ScanRange] {
+        let scanRangesPtr = zcashlc_suggest_scan_ranges(dbData.0, dbData.1, networkType.networkId)
+
+        guard let scanRangesPtr else {
+            throw ZcashError.rustSuggestScanRanges(lastErrorMessage(fallback: "`suggestScanRanges` failed with unknown error"))
+        }
+
+        defer { zcashlc_free_scan_ranges(scanRangesPtr) }
+
+        var scanRanges: [ScanRange] = []
+
+        for i in (0 ..< Int(scanRangesPtr.pointee.len)) {
+            let scanRange = scanRangesPtr.pointee.ptr.advanced(by: i).pointee
+
+            scanRanges.append(
+                ScanRange(
+                    range: Range(uncheckedBounds: (
+                        BlockHeight(scanRange.start),
+                        BlockHeight(scanRange.end)
+                    )),
+                    priority: scanRange.priority
+                )
+            )
+        }
+
+        return scanRanges
+    }
+
+    func scanBlocks(fromHeight: Int32, limit: UInt32 = 0) async throws {
+        let result = zcashlc_scan_blocks(fsBlockDbRoot.0, fsBlockDbRoot.1, dbData.0, dbData.1, fromHeight, limit, networkType.networkId)
 
         guard result != 0 else {
             throw ZcashError.rustScanBlocks(lastErrorMessage(fallback: "`scanBlocks` failed with unknown error"))
@@ -657,3 +731,11 @@ extension Array where Element == FFIBlockMeta {
         }
     }
 }
+
+extension Array where Element == FfiSubtreeRoot {
+    func deallocateElements() {
+        self.forEach { element in
+            element.root_hash_ptr.deallocate()
+        }
+    }
+}