[#1165] draft

draft

[#1165] Step 1 - Download note commitment tree data from lightwalletd

- code cleanup after draft

[#1165] Step 1 - Download note commitment tree data from lightwalletd

- UpdateSubtreeRootsAction added, ensuring the roots are downloaded and stored in the DB

[#1165] Step 1 - Download note commitment tree data from lightwalletd

- added ZcashError for putSaplingSubtreeRoots failure
- cleaned up action

[#1165] Step 1 - Download note commitment tree data from lightwalletd

- demo app config temporarily updated to Nighthawk server

[#1165] Step 1 - Download note commitment tree data from lightwalletd

- file header updated

[#1165] Step 1 - Download note commitment tree data from lightwalletd (#1174)

- demo app config cleaned up

[#1165] Step 1 - Download note commitment tree data from lightwalletd (#1174)

- offline tests fixed

Example/ZcashLightClientSample/ZcashLightClientSample/DemoAppConfig.swift

@@ -18,14 +18,14 @@ enum DemoAppConfig {
         let seed: [UInt8]
     }
 
-    static let host = ZcashSDK.isMainnet ? "lightwalletd.electriccoin.co" : "lightwalletd.testnet.electriccoin.co"
+    static let host = ZcashSDK.isMainnet ? "mainnet.lightwalletd.com" : "testnet.lightwalletd.com"
     static let port: Int = 9067
 
     static let defaultBirthdayHeight: BlockHeight = ZcashSDK.isMainnet ? 935000 : 1386000
     static let defaultSeed = try! Mnemonic.deterministicSeedBytes(from: """
     live combine flight accident slow soda mind bright absent bid hen shy decade biology amazing mix enlist ensure biology rhythm snap duty soap armor
     """)
-
+    
     static let otherSynchronizers: [SynchronizerInitData] = [
         SynchronizerInitData(
             alias: .custom("alt-sync-1"),

Sources/ZcashLightClientKit/Block/Actions/Action.swift

@@ -34,6 +34,7 @@ enum CBPState: CaseIterable {
     case idle
     case migrateLegacyCacheDB
     case validateServer
+    case updateSubtreeRoots
     case computeSyncControlData
     case download
     case scan

Sources/ZcashLightClientKit/Block/Actions/UpdateSubtreeRootsAction.swift

@@ -0,0 +1,68 @@
+//
+//  UpdateSubtreeRootsAction.swift
+//  
+//
+//  Created by Lukas Korba on 01.08.2023.
+//
+
+import Foundation
+
+final class UpdateSubtreeRootsAction {
+    let rustBackend: ZcashRustBackendWelding
+    let service: LightWalletService
+    let logger: Logger
+
+    init(container: DIContainer) {
+        service = container.resolve(LightWalletService.self)
+        rustBackend = container.resolve(ZcashRustBackendWelding.self)
+        logger = container.resolve(Logger.self)
+    }
+}
+
+extension UpdateSubtreeRootsAction: Action {
+    var removeBlocksCacheWhenFailed: Bool { false }
+
+    func run(with context: ActionContext, didUpdate: @escaping (CompactBlockProcessor.Event) async -> Void) async throws -> ActionContext {
+        var request = GetSubtreeRootsArg()
+        request.shieldedProtocol = .sapling
+        request.maxEntries = 65536
+
+        logger.info("Attempt to get subtree roots, this may fail because lightwalletd may not support DAG sync.")
+        let stream = service.getSubtreeRoots(request)
+
+        var roots: [SubtreeRoot] = []
+        var err: Error?
+
+        do {
+            for try await subtreeRoot in stream {
+                roots.append(subtreeRoot)
+            }
+        } catch {
+            logger.debug("getSubtreeRoots failed with error \(error.localizedDescription)")
+            err = error
+        }
+
+        // In case of error, the lightwalletd doesn't support DAG sync -> switching to linear sync.
+        // Likewise, no subtree roots results in switching to linear sync.
+        if err != nil || roots.isEmpty {
+            logger.info("DAG sync is not possible, switching to linear sync.")
+            await context.update(state: .computeSyncControlData)
+        } else {
+            logger.info("Sapling tree has \(roots.count) subtrees")
+            do {
+                try await rustBackend.putSaplingSubtreeRoots(startIndex: UInt64(request.startIndex), roots: roots)
+
+                // TODO: [#1167] Switching back to linear sync for now before step 3 & 4 are implemented
+                // https://github.com/zcash/ZcashLightClientKit/issues/1167
+                await context.update(state: .computeSyncControlData)
+            } catch {
+                logger.debug("putSaplingSubtreeRoots failed with error \(error.localizedDescription)")
+                throw ZcashError.compactBlockProcessorPutSaplingSubtreeRoots(error)
+            }
+        }
+
+        return context
+    }
+
+    func stop() async { }
+}

Sources/ZcashLightClientKit/Block/Actions/ValidateServerAction.swift

@@ -52,7 +52,7 @@ extension ValidateServerAction: Action {
             throw ZcashError.compactBlockProcessorWrongConsensusBranchId(localBranch, remoteBranchID)
         }
 
-        await context.update(state: .computeSyncControlData)
+        await context.update(state: .updateSubtreeRoots)
         return context
     }
 

Sources/ZcashLightClientKit/Block/CompactBlockProcessor.swift

@@ -214,6 +214,8 @@ actor CompactBlockProcessor {
                 action = MigrateLegacyCacheDBAction(container: container, configProvider: configProvider)
             case .validateServer:
                 action = ValidateServerAction(container: container, configProvider: configProvider)
+            case .updateSubtreeRoots:
+                action = UpdateSubtreeRootsAction(container: container)
             case .computeSyncControlData:
                 action = ComputeSyncControlDataAction(container: container, configProvider: configProvider)
             case .download:
@@ -585,6 +587,8 @@ extension CompactBlockProcessor {
             break
         case .validateServer:
             break
+        case .updateSubtreeRoots:
+            break
         case .computeSyncControlData:
             break
         case .download:

Sources/ZcashLightClientKit/Error/ZcashError.swift

@@ -58,6 +58,9 @@ public enum ZcashError: Equatable, Error {
     /// LightWalletService.blockStream failed.
     /// ZSRVC0000
     case serviceBlockStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getSubtreeRoots failed.
+    /// ZSRVC0009
+    case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
     /// SimpleConnectionProvider init of Connection failed.
     /// ZSCPC0001
     case simpleConnectionProvider(_ error: Error)
@@ -277,18 +280,22 @@ public enum ZcashError: Equatable, Error {
     /// ZRUST0045
     case rustGetTransparentReceiverInvalidReceiver
     /// Unable to allocate memory required to write blocks when calling ZcashRustBackend.putSaplingSubtreeRoots
+    /// sourcery: code="ZRUST0046"
     /// ZRUST0046
     case rustPutSaplingSubtreeRootsAllocationProblem
     /// Error from rust layer when calling ZcashRustBackend.putSaplingSubtreeRoots
     /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0047"
     /// ZRUST0047
     case rustPutSaplingSubtreeRoots(_ rustError: String)
     /// Error from rust layer when calling ZcashRustBackend.updateChainTip
     /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0048"
     /// ZRUST0048
     case rustUpdateChainTip(_ rustError: String)
     /// Error from rust layer when calling ZcashRustBackend.suggestScanRanges
     /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0049"
     /// ZRUST0049
     case rustSuggestScanRanges(_ rustError: String)
     /// SQLite query failed when fetching all accounts from the database.
@@ -527,6 +534,9 @@ public enum ZcashError: Equatable, Error {
     /// Rewind of DownloadBlockAction failed as no action is possible to unwrapp.
     /// ZCBPEO0018
     case compactBlockProcessorDownloadBlockActionRewind
+    /// Put sapling subtree roots to the DB failed.
+    /// ZCBPEO0019
+    case compactBlockProcessorPutSaplingSubtreeRoots(_ error: Error)
     /// The synchronizer is unprepared.
     /// ZSYNCO0001
     case synchronizerNotPrepared
@@ -563,6 +573,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchTransactionFailed: return "LightWalletService.fetchTransaction failed."
         case .serviceFetchUTXOsFailed: return "LightWalletService.fetchUTXOs failed."
         case .serviceBlockStreamFailed: return "LightWalletService.blockStream failed."
+        case .serviceSubtreeRootsStreamFailed: return "LightWalletService.getSubtreeRoots failed."
         case .simpleConnectionProvider: return "SimpleConnectionProvider init of Connection failed."
         case .saplingParamsInvalidSpendParams: return "Downloaded file with sapling spending parameters isn't valid."
         case .saplingParamsInvalidOutputParams: return "Downloaded file with sapling output parameters isn't valid."
@@ -620,7 +631,7 @@ public enum ZcashError: Equatable, Error {
         case .rustGetSaplingReceiverInvalidReceiver: return "Sapling receiver generated by rust layer is invalid when calling ZcashRustBackend.getSaplingReceiver"
         case .rustGetTransparentReceiverInvalidAddress: return "Error from rust layer when calling ZcashRustBackend.getTransparentReceiver"
         case .rustGetTransparentReceiverInvalidReceiver: return "Transparent receiver generated by rust layer is invalid when calling ZcashRustBackend.getTransparentReceiver"
-        case .rustPutSaplingSubtreeRootsAllocationProblem: return "Unable to allocate memory required to store subtree roots when calling ZcashRustBackend.putSaplingSubtreeRoots"
+        case .rustPutSaplingSubtreeRootsAllocationProblem: return "Unable to allocate memory required to write blocks when calling ZcashRustBackend.putSaplingSubtreeRoots"
         case .rustPutSaplingSubtreeRoots: return "Error from rust layer when calling ZcashRustBackend.putSaplingSubtreeRoots"
         case .rustUpdateChainTip: return "Error from rust layer when calling ZcashRustBackend.updateChainTip"
         case .rustSuggestScanRanges: return "Error from rust layer when calling ZcashRustBackend.suggestScanRanges"
@@ -699,6 +710,7 @@ public enum ZcashError: Equatable, Error {
         case .compactBlockProcessorChainName: return "Chain name does not match. Expected either 'test' or 'main'. This is probably an API or programming error."
         case .compactBlockProcessorConsensusBranchID: return "Consensus BranchIDs don't match this is probably an API or programming error."
         case .compactBlockProcessorDownloadBlockActionRewind: return "Rewind of DownloadBlockAction failed as no action is possible to unwrapp."
+        case .compactBlockProcessorPutSaplingSubtreeRoots: return "Put sapling subtree roots to the DB failed."
         case .synchronizerNotPrepared: return "The synchronizer is unprepared."
         case .synchronizerSendMemoToTransparentAddress: return "Memos can't be sent to transparent addresses."
         case .synchronizerShieldFundsInsuficientTransparentFunds: return "There is not enough transparent funds to cover fee for the shielding."
@@ -725,6 +737,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchTransactionFailed: return .serviceFetchTransactionFailed
         case .serviceFetchUTXOsFailed: return .serviceFetchUTXOsFailed
         case .serviceBlockStreamFailed: return .serviceBlockStreamFailed
+        case .serviceSubtreeRootsStreamFailed: return .serviceSubtreeRootsStreamFailed
         case .simpleConnectionProvider: return .simpleConnectionProvider
         case .saplingParamsInvalidSpendParams: return .saplingParamsInvalidSpendParams
         case .saplingParamsInvalidOutputParams: return .saplingParamsInvalidOutputParams
@@ -861,6 +874,7 @@ public enum ZcashError: Equatable, Error {
         case .compactBlockProcessorChainName: return .compactBlockProcessorChainName
         case .compactBlockProcessorConsensusBranchID: return .compactBlockProcessorConsensusBranchID
         case .compactBlockProcessorDownloadBlockActionRewind: return .compactBlockProcessorDownloadBlockActionRewind
+        case .compactBlockProcessorPutSaplingSubtreeRoots: return .compactBlockProcessorPutSaplingSubtreeRoots
         case .synchronizerNotPrepared: return .synchronizerNotPrepared
         case .synchronizerSendMemoToTransparentAddress: return .synchronizerSendMemoToTransparentAddress
         case .synchronizerShieldFundsInsuficientTransparentFunds: return .synchronizerShieldFundsInsuficientTransparentFunds

Sources/ZcashLightClientKit/Error/ZcashErrorCode.swift

@@ -39,6 +39,8 @@ public enum ZcashErrorCode: String {
     case serviceFetchUTXOsFailed = "ZSRVC0008"
     /// LightWalletService.blockStream failed.
     case serviceBlockStreamFailed = "ZSRVC0000"
+    /// LightWalletService.getSubtreeRoots failed.
+    case serviceSubtreeRootsStreamFailed = "ZSRVC0009"
     /// SimpleConnectionProvider init of Connection failed.
     case simpleConnectionProvider = "ZSCPC0001"
     /// Downloaded file with sapling spending parameters isn't valid.
@@ -311,6 +313,8 @@ public enum ZcashErrorCode: String {
     case compactBlockProcessorConsensusBranchID = "ZCBPEO0017"
     /// Rewind of DownloadBlockAction failed as no action is possible to unwrapp.
     case compactBlockProcessorDownloadBlockActionRewind = "ZCBPEO0018"
+    /// Put sapling subtree roots to the DB failed.
+    case compactBlockProcessorPutSaplingSubtreeRoots = "ZCBPEO0019"
     /// The synchronizer is unprepared.
     case synchronizerNotPrepared = "ZSYNCO0001"
     /// Memos can't be sent to transparent addresses.

Sources/ZcashLightClientKit/Error/ZcashErrorCodeDefinition.swift

@@ -77,6 +77,9 @@ enum ZcashErrorDefinition {
     /// LightWalletService.blockStream failed.
     // sourcery: code="ZSRVC0000"
     case serviceBlockStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getSubtreeRoots failed.
+    // sourcery: code="ZSRVC0009"
+    case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
 
     // MARK: SQLite connection
 
@@ -307,6 +310,21 @@ enum ZcashErrorDefinition {
     /// Transparent receiver generated by rust layer is invalid when calling ZcashRustBackend.getTransparentReceiver
     // sourcery: code="ZRUST0045"
     case rustGetTransparentReceiverInvalidReceiver
+    /// Unable to allocate memory required to write blocks when calling ZcashRustBackend.putSaplingSubtreeRoots
+    /// sourcery: code="ZRUST0046"
+    case rustPutSaplingSubtreeRootsAllocationProblem
+    /// Error from rust layer when calling ZcashRustBackend.putSaplingSubtreeRoots
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0047"
+    case rustPutSaplingSubtreeRoots(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.updateChainTip
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0048"
+    case rustUpdateChainTip(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.suggestScanRanges
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0049"
+    case rustSuggestScanRanges(_ rustError: String)
 
     // MARK: - Account DAO
 
@@ -592,6 +610,9 @@ enum ZcashErrorDefinition {
     /// Rewind of DownloadBlockAction failed as no action is possible to unwrapp.
     // sourcery: code="ZCBPEO0018"
     case compactBlockProcessorDownloadBlockActionRewind
+    /// Put sapling subtree roots to the DB failed.
+    // sourcery: code="ZCBPEO0019"
+    case compactBlockProcessorPutSaplingSubtreeRoots(_ error: Error)
 
     // MARK: - SDKSynchronizer
 

Sources/ZcashLightClientKit/Modules/Service/GRPC/LightWalletGRPCService.swift

@@ -262,6 +262,21 @@ extension LightWalletGRPCService: LightWalletService {
             }
         }
     }
+
+    func getSubtreeRoots(_ request: GetSubtreeRootsArg) -> AsyncThrowingStream<SubtreeRoot, Error> {
+        let stream = compactTxStreamer.getSubtreeRoots(request)
+        var iterator = stream.makeAsyncIterator()
+
+        return AsyncThrowingStream() {
+            do {
+                guard let subtreeRoot = try await iterator.next() else { return nil }
+                return subtreeRoot
+            } catch {
+                let serviceError = error.mapToServiceError()
+                throw ZcashError.serviceSubtreeRootsStreamFailed(serviceError)
+            }
+        }
+    }
 
     func closeConnection() {
         _ = channel.close()

Sources/ZcashLightClientKit/Modules/Service/LightWalletService.swift

@@ -193,4 +193,11 @@ protocol LightWalletService: AnyObject {
     ) -> AsyncThrowingStream<ZcashCompactBlock, Error>
 
     func closeConnection()
+
+    /// Returns a stream of information about roots of subtrees of the Sapling and Orchard
+    /// note commitment trees.
+    ///
+    /// - Parameters:
+    ///   - request: Request to send to GetSubtreeRoots.
+    func getSubtreeRoots(_ request: GetSubtreeRootsArg) -> AsyncThrowingStream<SubtreeRoot, Error>
 }

Sources/ZcashLightClientKit/Rust/ZcashRustBackendWelding.swift

@@ -170,6 +170,10 @@ protocol ZcashRustBackendWelding {
     /// - Throws: `rustRewindCacheToHeight` if rust layer returns error.
     func rewindCacheToHeight(height: Int32) async throws
 
+    func putSaplingSubtreeRoots(startIndex: UInt64, roots: [SubtreeRoot]) async throws
+
+    func updateChainTip(height: Int32) async throws
+
     /// Returns a list of suggested scan ranges based upon the current wallet state.
     ///
     /// This method should only be used in cases where the `CompactBlock` data that will be

Sources/ZcashLightClientKit/Synchronizer/SDKSynchronizer.swift

@@ -141,7 +141,7 @@ public class SDKSynchronizer: Synchronizer {
         if case .seedRequired = try await self.initializer.initialize(with: seed, viewingKeys: viewingKeys, walletBirthday: walletBirthday) {
             return .seedRequired
         }
-
+        
         await latestBlocksDataProvider.updateWalletBirthday(initializer.walletBirthday)
         await latestBlocksDataProvider.updateScannedData()
 

Tests/OfflineTests/CompactBlockProcessorActions/ValidateServerActionTests.swift

@@ -31,8 +31,8 @@ final class ValidateServerActionTests: ZcashTestCase {
             let nextContext = try await validateServerAction.run(with: .init(state: .validateServer)) { _ in }
             let nextState = await nextContext.state
             XCTAssertTrue(
-                nextState == .computeSyncControlData,
-                "nextContext after .validateServer is expected to be .computeSyncControlData but received \(nextState)"
+                nextState == .updateSubtreeRoots,
+                "nextContext after .validateServer is expected to be .updateSubtreeRoots but received \(nextState)"
             )
         } catch {
             XCTFail("testValidateServerAction_NextAction is not expected to fail. \(error)")