From e0150aacf335317a928fbc0663bdbe9df8cbc6b6 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Tue, 16 Feb 2021 13:19:27 -0800 Subject: [PATCH 01/48] wip --- src/distributed/assetstorage.mo | 109 +++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index f52680548f..86595bdb73 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -6,8 +6,45 @@ import Tree "mo:base/RBTree"; shared ({caller = creator}) actor class () { + public type BlobId = Text; + public type Key = Text; public type Path = Text; + public type Commit = Bool; public type Contents = Blob; + public type ContentEncoding = Text; + public type ContentType = Text; + public type Offset = Nat; + public type TotalLength = Nat; + + public type CreateAssetOperation = { + key: Key; + content_type: Text; + }; + public type SetAssetContentOperation = { + key: Key; + content_encoding: Text; + blob_id: BlobId; + }; + public type UnsetAssetContentOperation = { + key: Key; + content_encoding: Text; + }; + public type DeleteAssetOperation = { + key: Key; + }; + public type ClearOperation = { + }; + + public type BatchOperationKind = { + #create: CreateAssetOperation; + #set_content: SetAssetContentOperation; + #unset_content: UnsetAssetContentOperation; + + #delete: DeleteAssetOperation; + #clear: ClearOperation; + }; + + stable var authorized: [Principal] = [creator]; @@ -44,5 +81,75 @@ shared ({caller = creator}) actor class () { func isSafe(caller: Principal) : Bool { func eq(value: Principal): Bool = value == caller; Array.find(authorized, eq) != null - } + }; + + public query func get(arg:{ + key: Key; + accept_encodings: [Text] + }) : async ( { contents: Blob; content_type: Text; content_encoding: Text } ) { + throw Error.reject("get: not implemented"); + }; + + public func create_blobs( arg: { + blob_info: [ { + length: Nat32 + } ] + } ) : async ( { blob_ids: [BlobId] } ) { + throw Error.reject("create_blobs: not implemented"); + }; + + public func write_blob( arg: { + blob_id: BlobId; + offset: Nat32; + contents: Blob + } ) : async () { + throw Error.reject("write_blob: not implemented"); + }; + + public func batch(ops: [BatchOperationKind]) : async() { + throw Error.reject("batch: not implemented"); + }; + + public func create_asset(op: CreateAssetOperation) : async () { + throw Error.reject("create_asset: not implemented"); + }; + + public func set_asset_content(op: SetAssetContentOperation) : async () { + throw Error.reject("set_asset_content: not implemented"); + }; + + public func unset_asset_content(op: UnsetAssetContentOperation) : async () { + throw Error.reject("unset_asset_content: not implemented"); + }; + + public func delete_asset(op: DeleteAssetOperation) : async () { + throw Error.reject("delete_asset: not implemented"); + }; + + public func clear(op: ClearOperation) : async () { + throw Error.reject("clear: not implemented"); + }; + + public func create(arg:{ + path: Path; + contents: Contents; + total_length: TotalLength; + content_type: ContentType; + content_encoding: ContentEncoding; + commit: Commit}) : async () { + throw Error.reject("not implemented"); + }; + + public func write(path: Path, offset: Offset, contents: Contents) : async () { + throw Error.reject("not implemented"); + }; + + public func commit(path: Path) : async () { + throw Error.reject("not implemented"); + }; + + public func commit_many(paths: [Path]) : async () { + throw Error.reject("not implemented"); + }; + }; From 42abc48d963e374228bd82f0ea4b6f4c443421a4 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 17 Feb 2021 13:39:24 -0800 Subject: [PATCH 02/48] checkpoint stable memory ffs --- src/distributed/assetstorage.mo | 45 +++++++++++++++------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 86595bdb73..04a87a01ce 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -1,4 +1,5 @@ import Error "mo:base/Error"; +import H "mo:base/HashMap"; import Iter "mo:base/Iter"; import Array "mo:base/Array"; import Text "mo:base/Text"; @@ -16,6 +17,7 @@ shared ({caller = creator}) actor class () { public type Offset = Nat; public type TotalLength = Nat; + public type CreateAssetOperation = { key: Key; content_type: Text; @@ -50,6 +52,21 @@ shared ({caller = creator}) actor class () { let db: Tree.RBTree = Tree.RBTree(Text.compare); + type Asset = { + content_type: Text; + }; + + stable var asset_entries : [(Key, Asset)] = []; + let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); + + system func preupgrade() { + asset_entries := Iter.toArray(assets.entries()); + }; + + system func postupgrade() { + asset_entries := []; + }; + public shared ({ caller }) func authorize(other: Principal) : async () { if (isSafe(caller)) { authorized := Array.append(authorized, [other]); @@ -87,7 +104,10 @@ shared ({caller = creator}) actor class () { key: Key; accept_encodings: [Text] }) : async ( { contents: Blob; content_type: Text; content_encoding: Text } ) { - throw Error.reject("get: not implemented"); + switch (assets.get(arg.key)) { + case null throw Error.reject("not found"); + case (?asset) throw Error.reject("found but not implemented"); + }; }; public func create_blobs( arg: { @@ -129,27 +149,4 @@ shared ({caller = creator}) actor class () { public func clear(op: ClearOperation) : async () { throw Error.reject("clear: not implemented"); }; - - public func create(arg:{ - path: Path; - contents: Contents; - total_length: TotalLength; - content_type: ContentType; - content_encoding: ContentEncoding; - commit: Commit}) : async () { - throw Error.reject("not implemented"); - }; - - public func write(path: Path, offset: Offset, contents: Contents) : async () { - throw Error.reject("not implemented"); - }; - - public func commit(path: Path) : async () { - throw Error.reject("not implemented"); - }; - - public func commit_many(paths: [Path]) : async () { - throw Error.reject("not implemented"); - }; - }; From 38fedbab9241e9b5155c46e08de4cf338441cb74 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 17 Feb 2021 14:53:32 -0800 Subject: [PATCH 03/48] batch expiry map --- src/distributed/assetstorage.mo | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 04a87a01ce..f436cb862e 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -1,8 +1,10 @@ +import Array "mo:base/Array"; import Error "mo:base/Error"; import H "mo:base/HashMap"; +import Int "mo:base/Int"; import Iter "mo:base/Iter"; -import Array "mo:base/Array"; import Text "mo:base/Text"; +import Time "mo:base/Time"; import Tree "mo:base/RBTree"; shared ({caller = creator}) actor class () { @@ -67,6 +69,23 @@ shared ({caller = creator}) actor class () { asset_entries := []; }; + // blobs data doesn't need to be stable + type BlobInfo = { + batch_id: Nat; + blob: Blob; + }; + var next_blob_id = 1; + let blobs = H.HashMap(7, Text.equal, Text.hash); + func alloc_blob_id() : Nat { + let result = next_blob_id; + next_blob_id += 1; + result + }; + + var next_batch_id = 1; + type Time = Int; + let batch_expiry = H.HashMap(7, Int.equal, Int.hash); + public shared ({ caller }) func authorize(other: Principal) : async () { if (isSafe(caller)) { authorized := Array.append(authorized, [other]); From 0362051e0fb5d0d2d2b7cb55488d93d43b456085 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 17 Feb 2021 15:18:57 -0800 Subject: [PATCH 04/48] create blobs compiles --- src/distributed/assetstorage.mo | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index f436cb862e..789d5bd456 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -3,12 +3,14 @@ import Error "mo:base/Error"; import H "mo:base/HashMap"; import Int "mo:base/Int"; import Iter "mo:base/Iter"; +import Result "mo:base/Result"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; shared ({caller = creator}) actor class () { + public type BatchId = Int; public type BlobId = Text; public type Key = Text; public type Path = Text; @@ -76,16 +78,25 @@ shared ({caller = creator}) actor class () { }; var next_blob_id = 1; let blobs = H.HashMap(7, Text.equal, Text.hash); - func alloc_blob_id() : Nat { + func alloc_blob_id() : BlobId { let result = next_blob_id; next_blob_id += 1; - result + Int.toText(result) }; + // We track when each group of blobs should expire, + // so that they don't consume space after an interrupted install. + let BATCH_EXPIRY_NANOS = 5 * 1000 * 1000; var next_batch_id = 1; type Time = Int; let batch_expiry = H.HashMap(7, Int.equal, Int.hash); + func start_batch(): BatchId { + let batch_id = next_batch_id; + next_batch_id += 1; + let expires = Time.now() + BATCH_EXPIRY_NANOS; + }; + public shared ({ caller }) func authorize(other: Principal) : async () { if (isSafe(caller)) { authorized := Array.append(authorized, [other]); @@ -129,11 +140,21 @@ shared ({caller = creator}) actor class () { }; }; + func createBlob(batchId: BatchId, length: Nat32) : Result.Result { + #ok(alloc_blob_id()) + }; + public func create_blobs( arg: { blob_info: [ { length: Nat32 } ] } ) : async ( { blob_ids: [BlobId] } ) { + let batch_id = start_batch(); + + let createBlobInBatch = func (arg: { length: Nat32 }) : Result.Result { + createBlob(batch_id, arg.length) + }; + throw Error.reject("create_blobs: not implemented"); }; From 4d270f05f0b6fcb959eacdf5c27c8d6fa8047748 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 17 Feb 2021 17:20:24 -0800 Subject: [PATCH 05/48] create blobs --- src/distributed/assetstorage.mo | 40 ++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 789d5bd456..bbc24d9948 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -3,6 +3,8 @@ import Error "mo:base/Error"; import H "mo:base/HashMap"; import Int "mo:base/Int"; import Iter "mo:base/Iter"; +import Nat8 "mo:base/Nat8"; +import Nat32 "mo:base/Nat32"; import Result "mo:base/Result"; import Text "mo:base/Text"; import Time "mo:base/Time"; @@ -10,7 +12,7 @@ import Tree "mo:base/RBTree"; shared ({caller = creator}) actor class () { - public type BatchId = Int; + public type BatchId = Nat; public type BlobId = Text; public type Key = Text; public type Path = Text; @@ -71,13 +73,14 @@ shared ({caller = creator}) actor class () { asset_entries := []; }; - // blobs data doesn't need to be stable - type BlobInfo = { - batch_id: Nat; - blob: Blob; + // blob data doesn't need to be stable + class BlobBuffer(initBatchId: Nat, initBlob: [var Nat8]) { + let batchId = initBatchId; + let blob = initBlob; }; + var next_blob_id = 1; - let blobs = H.HashMap(7, Text.equal, Text.hash); + let blobs = H.HashMap(7, Text.equal, Text.hash); func alloc_blob_id() : BlobId { let result = next_blob_id; next_blob_id += 1; @@ -95,6 +98,7 @@ shared ({caller = creator}) actor class () { let batch_id = next_batch_id; next_batch_id += 1; let expires = Time.now() + BATCH_EXPIRY_NANOS; + batch_id }; public shared ({ caller }) func authorize(other: Principal) : async () { @@ -141,13 +145,24 @@ shared ({caller = creator}) actor class () { }; func createBlob(batchId: BatchId, length: Nat32) : Result.Result { - #ok(alloc_blob_id()) + let blobId = alloc_blob_id(); + + let blob = Array.init(Nat32.toNat(length), 0); + let blobBuffer = BlobBuffer(batchId, blob); + + blobs.put(blobId, blobBuffer); + + #ok(blobId) }; + //type BlobParameters = { + // length: Nat32 + //}; + //type CreateBlobsResult = { + // blob_ids: [BlobId] + //}; public func create_blobs( arg: { - blob_info: [ { - length: Nat32 - } ] + blob_info: [ { length: Nat32 } ] } ) : async ( { blob_ids: [BlobId] } ) { let batch_id = start_batch(); @@ -155,7 +170,10 @@ shared ({caller = creator}) actor class () { createBlob(batch_id, arg.length) }; - throw Error.reject("create_blobs: not implemented"); + switch(Array.mapResult<{length: Nat32}, BlobId, Text>(arg.blob_info, createBlobInBatch)) { + case (#ok(ids)) { { blob_ids = ids } }; + case (#err(err)) throw Error.reject(err); + } }; public func write_blob( arg: { From 2d669d0ad7a48ef64e94c2e36d03c05d9ae38f0b Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 17 Feb 2021 17:51:33 -0800 Subject: [PATCH 06/48] first pass writeBlob --- src/distributed/assetstorage.mo | 73 ++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index bbc24d9948..aaf6204cd5 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -3,6 +3,7 @@ import Error "mo:base/Error"; import H "mo:base/HashMap"; import Int "mo:base/Int"; import Iter "mo:base/Iter"; +import Nat "mo:base/Nat"; import Nat8 "mo:base/Nat8"; import Nat32 "mo:base/Nat32"; import Result "mo:base/Result"; @@ -73,11 +74,27 @@ shared ({caller = creator}) actor class () { asset_entries := []; }; - // blob data doesn't need to be stable - class BlobBuffer(initBatchId: Nat, initBlob: [var Nat8]) { - let batchId = initBatchId; - let blob = initBlob; + // blob data doesn't need to be stable + class BlobBuffer(initBatchId: Nat, initBuffer: [var Word8]) { + let batchId = initBatchId; + let buffer = initBuffer; + + public func setData(offset: Nat32, data: Blob): Result.Result<(), Text> { + var index: Nat = Nat32.toNat(offset); + + if (index + data.size() > buffer.size()) { + #err("overflow: offset " # Nat32.toText(offset) # + " + data size " # Nat.toText(data.size()) # + " exceeds blob size of " # Nat.toText(buffer.size())) + } else { + for (b in data.bytes()) { + buffer[index] := b; + index += 1; + }; + #ok() + } }; + }; var next_blob_id = 1; let blobs = H.HashMap(7, Text.equal, Text.hash); @@ -147,7 +164,7 @@ shared ({caller = creator}) actor class () { func createBlob(batchId: BatchId, length: Nat32) : Result.Result { let blobId = alloc_blob_id(); - let blob = Array.init(Nat32.toNat(length), 0); + let blob = Array.init(Nat32.toNat(length), 0); let blobBuffer = BlobBuffer(batchId, blob); blobs.put(blobId, blobBuffer); @@ -161,46 +178,54 @@ shared ({caller = creator}) actor class () { //type CreateBlobsResult = { // blob_ids: [BlobId] //}; - public func create_blobs( arg: { - blob_info: [ { length: Nat32 } ] - } ) : async ( { blob_ids: [BlobId] } ) { - let batch_id = start_batch(); + public func createBlobs( arg: { + blobInfo: [ { length: Nat32 } ] + } ) : async ( { blobIds: [BlobId] } ) { + let batchId = start_batch(); let createBlobInBatch = func (arg: { length: Nat32 }) : Result.Result { - createBlob(batch_id, arg.length) + createBlob(batchId, arg.length) }; - switch(Array.mapResult<{length: Nat32}, BlobId, Text>(arg.blob_info, createBlobInBatch)) { - case (#ok(ids)) { { blob_ids = ids } }; + switch (Array.mapResult<{length: Nat32}, BlobId, Text>(arg.blobInfo, createBlobInBatch)) { + case (#ok(ids)) { { blobIds = ids } }; case (#err(err)) throw Error.reject(err); } }; - public func write_blob( arg: { - blob_id: BlobId; - offset: Nat32; - contents: Blob - } ) : async () { - throw Error.reject("write_blob: not implemented"); - }; + public func writeBlob( arg: { + blobId: BlobId; + offset: Nat32; + contents: Blob + } ) : async () { + switch (blobs.get(arg.blobId)) { + case null throw Error.reject("Blob not found"); + case (?blobBuffer) { + switch (blobBuffer.setData(arg.offset, arg.contents)) { + case (#ok) {}; + case (#err(text)) throw Error.reject(text); + } + }; + } + }; - public func batch(ops: [BatchOperationKind]) : async() { + public func batch(ops: [BatchOperationKind]) : async () { throw Error.reject("batch: not implemented"); }; - public func create_asset(op: CreateAssetOperation) : async () { + public func createAsset(op: CreateAssetOperation) : async () { throw Error.reject("create_asset: not implemented"); }; - public func set_asset_content(op: SetAssetContentOperation) : async () { + public func setAssetContent(op: SetAssetContentOperation) : async () { throw Error.reject("set_asset_content: not implemented"); }; - public func unset_asset_content(op: UnsetAssetContentOperation) : async () { + public func unsetAssetContent(op: UnsetAssetContentOperation) : async () { throw Error.reject("unset_asset_content: not implemented"); }; - public func delete_asset(op: DeleteAssetOperation) : async () { + public func deleteAsset(op: DeleteAssetOperation) : async () { throw Error.reject("delete_asset: not implemented"); }; From 8e6f956b10ae561b2f25468b4f611b6988585cef Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 18 Feb 2021 10:21:17 -0800 Subject: [PATCH 07/48] motoko name conventions --- src/distributed/assetstorage.mo | 114 +++++++++++++++----------------- 1 file changed, 55 insertions(+), 59 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index aaf6204cd5..00091bff8a 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -1,4 +1,5 @@ import Array "mo:base/Array"; +import Buffer "mo:base/Buffer"; import Error "mo:base/Error"; import H "mo:base/HashMap"; import Int "mo:base/Int"; @@ -96,27 +97,28 @@ shared ({caller = creator}) actor class () { }; }; - var next_blob_id = 1; - let blobs = H.HashMap(7, Text.equal, Text.hash); - func alloc_blob_id() : BlobId { - let result = next_blob_id; - next_blob_id += 1; - Int.toText(result) - }; + var nextBlobId = 1; + let blobs = H.HashMap(7, Text.equal, Text.hash); - // We track when each group of blobs should expire, - // so that they don't consume space after an interrupted install. - let BATCH_EXPIRY_NANOS = 5 * 1000 * 1000; - var next_batch_id = 1; - type Time = Int; - let batch_expiry = H.HashMap(7, Int.equal, Int.hash); + func allocBlobId() : BlobId { + let result = nextBlobId; + nextBlobId += 1; + Int.toText(result) + }; - func start_batch(): BatchId { - let batch_id = next_batch_id; - next_batch_id += 1; - let expires = Time.now() + BATCH_EXPIRY_NANOS; - batch_id - }; + // We track when each group of blobs should expire, + // so that they don't consume space after an interrupted install. + let BATCH_EXPIRY_NANOS = 5 * 1000 * 1000; + var next_batch_id = 1; + type Time = Int; + let batch_expiry = H.HashMap(7, Int.equal, Int.hash); + + func startBatch(): BatchId { + let batch_id = next_batch_id; + next_batch_id += 1; + let expires = Time.now() + BATCH_EXPIRY_NANOS; + batch_id + }; public shared ({ caller }) func authorize(other: Principal) : async () { if (isSafe(caller)) { @@ -151,54 +153,48 @@ shared ({caller = creator}) actor class () { Array.find(authorized, eq) != null }; - public query func get(arg:{ - key: Key; - accept_encodings: [Text] - }) : async ( { contents: Blob; content_type: Text; content_encoding: Text } ) { - switch (assets.get(arg.key)) { - case null throw Error.reject("not found"); - case (?asset) throw Error.reject("found but not implemented"); - }; + public query func get(arg:{ + key: Key; + accept_encodings: [Text] + }) : async ( { contents: Blob; content_type: Text; content_encoding: Text } ) { + switch (assets.get(arg.key)) { + case null throw Error.reject("not found"); + case (?asset) throw Error.reject("found but not implemented"); }; + }; - func createBlob(batchId: BatchId, length: Nat32) : Result.Result { - let blobId = alloc_blob_id(); + func createBlob(batchId: BatchId, length: Nat32) : Result.Result { + let blobId = allocBlobId(); - let blob = Array.init(Nat32.toNat(length), 0); - let blobBuffer = BlobBuffer(batchId, blob); + let blob = Array.init(Nat32.toNat(length), 0); + let blobBuffer = BlobBuffer(batchId, blob); - blobs.put(blobId, blobBuffer); + blobs.put(blobId, blobBuffer); - #ok(blobId) - }; - - //type BlobParameters = { - // length: Nat32 - //}; - //type CreateBlobsResult = { - // blob_ids: [BlobId] - //}; - public func createBlobs( arg: { - blobInfo: [ { length: Nat32 } ] - } ) : async ( { blobIds: [BlobId] } ) { - let batchId = start_batch(); + #ok(blobId) + }; - let createBlobInBatch = func (arg: { length: Nat32 }) : Result.Result { - createBlob(batchId, arg.length) - }; + public func create_blobs( arg: { + blob_info: [ { length: Nat32 } ] + } ) : async ( { blob_ids: [BlobId] } ) { + let batchId = startBatch(); - switch (Array.mapResult<{length: Nat32}, BlobId, Text>(arg.blobInfo, createBlobInBatch)) { - case (#ok(ids)) { { blobIds = ids } }; - case (#err(err)) throw Error.reject(err); - } + let createBlobInBatch = func (arg: { length: Nat32 }) : Result.Result { + createBlob(batchId, arg.length) }; - public func writeBlob( arg: { - blobId: BlobId; + switch (Array.mapResult<{length: Nat32}, BlobId, Text>(arg.blob_info, createBlobInBatch)) { + case (#ok(ids)) { { blob_ids = ids } }; + case (#err(err)) throw Error.reject(err); + } + }; + + public func write_blob( arg: { + blob_id: BlobId; offset: Nat32; contents: Blob } ) : async () { - switch (blobs.get(arg.blobId)) { + switch (blobs.get(arg.blob_id)) { case null throw Error.reject("Blob not found"); case (?blobBuffer) { switch (blobBuffer.setData(arg.offset, arg.contents)) { @@ -213,19 +209,19 @@ shared ({caller = creator}) actor class () { throw Error.reject("batch: not implemented"); }; - public func createAsset(op: CreateAssetOperation) : async () { + public func create_asset(op: CreateAssetOperation) : async () { throw Error.reject("create_asset: not implemented"); }; - public func setAssetContent(op: SetAssetContentOperation) : async () { + public func set_asset_content(op: SetAssetContentOperation) : async () { throw Error.reject("set_asset_content: not implemented"); }; - public func unsetAssetContent(op: UnsetAssetContentOperation) : async () { + public func unset_asset_content(op: UnsetAssetContentOperation) : async () { throw Error.reject("unset_asset_content: not implemented"); }; - public func deleteAsset(op: DeleteAssetOperation) : async () { + public func delete_asset(op: DeleteAssetOperation) : async () { throw Error.reject("delete_asset: not implemented"); }; From c30fe2d57292907afb94551ad29fe4f9423b05e1 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 18 Feb 2021 14:06:12 -0800 Subject: [PATCH 08/48] ugh --- src/distributed/StableHashMap.mo | 18 ++++++ src/distributed/assetstorage.mo | 94 +++++++++++++++++++++++++++++--- 2 files changed, 103 insertions(+), 9 deletions(-) create mode 100644 src/distributed/StableHashMap.mo diff --git a/src/distributed/StableHashMap.mo b/src/distributed/StableHashMap.mo new file mode 100644 index 0000000000..ba771c0f6c --- /dev/null +++ b/src/distributed/StableHashMap.mo @@ -0,0 +1,18 @@ +import Prim "mo:prim"; +import P "mo:base/Prelude"; +import A "mo:base/Array"; +import Hash "mo:base/Hash"; +import Iter "mo:base/Iter"; +import AssocList "mo:base/AssocList"; + +module { + +// key-val list type +type KVs = AssocList.AssocList; + +public class StableHashMap() { + var table : [var KVs] = [var]; + var _count : Nat = 0; +}; + +} \ No newline at end of file diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 00091bff8a..409ba0dc2f 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -8,6 +8,7 @@ import Nat "mo:base/Nat"; import Nat8 "mo:base/Nat8"; import Nat32 "mo:base/Nat32"; import Result "mo:base/Result"; +import SHM "StableHashMap"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; @@ -60,9 +61,19 @@ shared ({caller = creator}) actor class () { let db: Tree.RBTree = Tree.RBTree(Text.compare); - type Asset = { - content_type: Text; - }; + type AssetEncoding = { + contentEncoding: Text; + content: [Word8]; + }; + + type Asset = { + contentType: Text; + encodings: SHM.StableHashMap; + }; + + func getAssetEncoding(asset : Asset, acceptEncodings : [Text]) : ?AssetEncoding { + null + }; stable var asset_entries : [(Key, Asset)] = []; let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); @@ -95,6 +106,11 @@ shared ({caller = creator}) actor class () { #ok() } }; + + public func takeBuffer() : [Word8] { + let x = Array.freeze(buffer); + x + }; }; var nextBlobId = 1; @@ -106,6 +122,17 @@ shared ({caller = creator}) actor class () { Int.toText(result) }; + func takeBlob(blobId: BlobId) : ?[Word8] { + switch (blobs.remove(blobId)) { + case null null; + case (?blobBuffer) { + let b: [Word8] = blobBuffer.takeBuffer(); + //let blob : Blob = b; + ?b + } + } + }; + // We track when each group of blobs should expire, // so that they don't consume space after an interrupted install. let BATCH_EXPIRY_NANOS = 5 * 1000 * 1000; @@ -152,17 +179,40 @@ shared ({caller = creator}) actor class () { func eq(value: Principal): Bool = value == caller; Array.find(authorized, eq) != null }; + public query func get2() : async( { contents: Blob } ) { + throw Error.reject("nyi"); + }; public query func get(arg:{ key: Key; accept_encodings: [Text] - }) : async ( { contents: Blob; content_type: Text; content_encoding: Text } ) { + }) : async ( { contents: [Word8]; content_type: Text; content_encoding: Text } ) { switch (assets.get(arg.key)) { case null throw Error.reject("not found"); - case (?asset) throw Error.reject("found but not implemented"); + case (?asset) { + switch (getAssetEncoding(asset, arg.accept_encodings)) { + case null throw Error.reject("no such encoding"); + case (?encoding) { + { + contents = encoding.content; + content_type = asset.contentType; + content_encoding = encoding.contentEncoding; + } + } + }; + }; }; }; + //func arrayToBlob(a : [Word8]) : Blob { + // a + //}; + + + //func nat8ArrayToBlob(a : [Nat8]) : Blob { + // a + //}; + func createBlob(batchId: BatchId, length: Nat32) : Result.Result { let blobId = allocBlobId(); @@ -209,13 +259,39 @@ shared ({caller = creator}) actor class () { throw Error.reject("batch: not implemented"); }; - public func create_asset(op: CreateAssetOperation) : async () { - throw Error.reject("create_asset: not implemented"); - }; + public func create_asset(arg: CreateAssetOperation) : async () { + switch (assets.get(arg.key)) { + case null { + let asset : Asset = { + contentType = arg.content_type; + encodings = SHM.StableHashMap(); + }; + assets.put( (arg.key, asset) ); + }; + case (?asset) { + if (asset.contentType != arg.content_type) + throw Error.reject("create_asset: content type mismatch"); - public func set_asset_content(op: SetAssetContentOperation) : async () { + } + } + }; + + public func set_asset_content(arg: SetAssetContentOperation) : async () { + switch (assets.get(arg.key), takeBlob(arg.blob_id)) { + case (null,null) throw Error.reject("Asset and Blob not found"); + case (null,?blob) throw Error.reject("Asset not found"); + case (?asset,null) throw Error.reject("Blob not found"); + case (?asset,?blob) { + let encoding : AssetEncoding = { + contentEncoding = arg.content_encoding; + content = blob; + }; + + //asset.encodings.put((arg.content_encoding, encoding)); throw Error.reject("set_asset_content: not implemented"); + }; }; + }; public func unset_asset_content(op: UnsetAssetContentOperation) : async () { throw Error.reject("unset_asset_content: not implemented"); From 77de230afe50e154dc1410ffe120ebfab1563e2f Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 18 Feb 2021 14:15:45 -0800 Subject: [PATCH 09/48] Word8 -> Nat8 --- src/distributed/assetstorage.mo | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 409ba0dc2f..a463c1a3c2 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -12,6 +12,7 @@ import SHM "StableHashMap"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; +import Word8 "mo:base/Word8"; shared ({caller = creator}) actor class () { @@ -63,7 +64,7 @@ shared ({caller = creator}) actor class () { type AssetEncoding = { contentEncoding: Text; - content: [Word8]; + content: [Nat8]; }; type Asset = { @@ -87,7 +88,7 @@ shared ({caller = creator}) actor class () { }; // blob data doesn't need to be stable - class BlobBuffer(initBatchId: Nat, initBuffer: [var Word8]) { + class BlobBuffer(initBatchId: Nat, initBuffer: [var Nat8]) { let batchId = initBatchId; let buffer = initBuffer; @@ -100,14 +101,14 @@ shared ({caller = creator}) actor class () { " exceeds blob size of " # Nat.toText(buffer.size())) } else { for (b in data.bytes()) { - buffer[index] := b; + buffer[index] := Nat8.fromNat(Word8.toNat(b)); index += 1; }; #ok() } }; - public func takeBuffer() : [Word8] { + public func takeBuffer() : [Nat8] { let x = Array.freeze(buffer); x }; @@ -122,11 +123,11 @@ shared ({caller = creator}) actor class () { Int.toText(result) }; - func takeBlob(blobId: BlobId) : ?[Word8] { + func takeBlob(blobId: BlobId) : ?[Nat8] { switch (blobs.remove(blobId)) { case null null; case (?blobBuffer) { - let b: [Word8] = blobBuffer.takeBuffer(); + let b: [Nat8] = blobBuffer.takeBuffer(); //let blob : Blob = b; ?b } @@ -186,7 +187,7 @@ shared ({caller = creator}) actor class () { public query func get(arg:{ key: Key; accept_encodings: [Text] - }) : async ( { contents: [Word8]; content_type: Text; content_encoding: Text } ) { + }) : async ( { contents: [Nat8]; content_type: Text; content_encoding: Text } ) { switch (assets.get(arg.key)) { case null throw Error.reject("not found"); case (?asset) { @@ -216,7 +217,7 @@ shared ({caller = creator}) actor class () { func createBlob(batchId: BatchId, length: Nat32) : Result.Result { let blobId = allocBlobId(); - let blob = Array.init(Nat32.toNat(length), 0); + let blob = Array.init(Nat32.toNat(length), 0); let blobBuffer = BlobBuffer(batchId, blob); blobs.put(blobId, blobBuffer); From 5c6f39e84ecf28770c68f0572b4c6f1650ea74af Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 19 Feb 2021 11:48:14 -0800 Subject: [PATCH 10/48] stable hash map stuff; start on first test --- e2e/tests-dfx/assetscanister.bash | 33 ++++++++++++++++++ src/distributed/StableHashMap.mo | 58 +++++++++++++++++++++++++++++-- src/distributed/assetstorage.mo | 29 ++++++++++++---- 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index ac2f32b557..c478702487 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -45,3 +45,36 @@ teardown() { HOME=. assert_command_fail dfx canister call --update e2e_project_assets store '("index.js", vec { 1; 2; 3; })' } + +@test 'new asset canister interface' { + install_asset assetscanister + + dfx_start + dfx canister create --all + dfx build + dfx canister install e2e_project_assets + + assert_command dfx canister call --query e2e_project_assets retrieve '("binary/noise.txt")' --output idl + assert_eq '(blob "\b8\01 \80\0aw12 \00xy\0aKL\0b\0ajk")' + + assert_command dfx canister call --query e2e_project_assets retrieve '("text-with-newlines.txt")' --output idl + assert_eq '(blob "cherries\0ait'\''s cherry season\0aCHERRIES")' + + assert_command dfx canister call --update e2e_project_assets store '("AA", blob "hello, world!")' + assert_eq '()' + assert_command dfx canister call --update e2e_project_assets store '("B", vec { 88; 87; 86; })' + assert_eq '()' + + assert_command dfx canister call --query e2e_project_assets retrieve '("B")' --output idl + assert_eq '(blob "XWV")' + + assert_command dfx canister call --query e2e_project_assets retrieve '("AA")' --output idl + assert_eq '(blob "hello, world!")' + + assert_command dfx canister call --query e2e_project_assets retrieve '("B")' --output idl + assert_eq '(blob "XWV")' + + assert_command_fail dfx canister call --query e2e_project_assets retrieve '("C")' + + HOME=. assert_command_fail dfx canister call --update e2e_project_assets store '("index.js", vec { 1; 2; 3; })' +} \ No newline at end of file diff --git a/src/distributed/StableHashMap.mo b/src/distributed/StableHashMap.mo index ba771c0f6c..2bc3390b34 100644 --- a/src/distributed/StableHashMap.mo +++ b/src/distributed/StableHashMap.mo @@ -11,8 +11,62 @@ module { type KVs = AssocList.AssocList; public class StableHashMap() { - var table : [var KVs] = [var]; - var _count : Nat = 0; + public var table : [var KVs] = [var]; + public var _count : Nat = 0; +}; + +public class StableHashMapManipulator( + initCapacity: Nat, + keyEq: (K,K) -> Bool, + keyHash: K -> Hash.Hash +) { + + /// Insert the value `v` at key `k`. Overwrites an existing entry with key `k` + public func put(m: StableHashMap, k : K, v : V) = ignore replace(m, k, v); + + /// Insert the value `v` at key `k` and returns the previous value stored at + /// `k` or null if it didn't exist. + public func replace(m: StableHashMap, k:K, v:V) : ?V { + if (m._count >= m.table.size()) { + let size = + if (m._count == 0) { + if (initCapacity > 0) { + initCapacity + } else { + 1 + } + } else { + m.table.size() * 2; + }; + let table2 = A.init>(size, null); + for (i in m.table.keys()) { + var kvs = m.table[i]; + label moveKeyVals : () + loop { + switch kvs { + case null { break moveKeyVals }; + case (?((k, v), kvsTail)) { + let h = Prim.word32ToNat(keyHash(k)); + let pos2 = h % table2.size(); + table2[pos2] := ?((k,v), table2[pos2]); + kvs := kvsTail; + }; + } + }; + }; + m.table := table2; + }; + let h = Prim.word32ToNat(keyHash(k)); + let pos = h % m.table.size(); + let (kvs2, ov) = AssocList.replace(m.table[pos], k, keyEq, ?v); + m.table[pos] := kvs2; + switch(ov){ + case null { m._count += 1 }; + case _ {} + }; + ov + }; + }; } \ No newline at end of file diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index a463c1a3c2..1f0eaf00cc 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -78,6 +78,8 @@ shared ({caller = creator}) actor class () { stable var asset_entries : [(Key, Asset)] = []; let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); + let assets_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); + let encodings_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); system func preupgrade() { asset_entries := Iter.toArray(assets.entries()); @@ -164,12 +166,27 @@ shared ({caller = creator}) actor class () { }; }; - public query func retrieve(path : Path) : async Contents { - switch (db.get(path)) { - case null throw Error.reject("not found"); - case (?contents) contents; - }; + //public query func retrieveX(path : Path) : async Contents { + // let arg = { + // key = path; + // accept_encodings = []; + // }; + // let x = get(arg); + // x.contents + //}; + public query func retrieve(path : Path) : async [Nat8] { + switch (assets.get(path)) { + case null throw Error.reject("not found"); + case (?asset) { + switch (getAssetEncoding(asset, [])) { + case null throw Error.reject("no such encoding"); + case (?encoding) { + encoding.content + } + }; + }; }; + }; public query func list() : async [Path] { let iter = Iter.map<(Path, Contents), Path>(db.entries(), func (path, _) = path); @@ -288,7 +305,7 @@ shared ({caller = creator}) actor class () { content = blob; }; - //asset.encodings.put((arg.content_encoding, encoding)); + encodings_manipulator.put(asset.encodings, arg.content_encoding, encoding); throw Error.reject("set_asset_content: not implemented"); }; }; From c4a7d87b932152b7c2d871b05c1b0bb5870ed14f Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 19 Feb 2021 12:47:01 -0800 Subject: [PATCH 11/48] SHM.get --- src/distributed/StableHashMap.mo | 12 ++++++++++++ src/distributed/assetstorage.mo | 10 +++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/distributed/StableHashMap.mo b/src/distributed/StableHashMap.mo index 2bc3390b34..08cf77a2c9 100644 --- a/src/distributed/StableHashMap.mo +++ b/src/distributed/StableHashMap.mo @@ -21,6 +21,18 @@ public class StableHashMapManipulator( keyHash: K -> Hash.Hash ) { + /// Gets the entry with the key `k` and returns its associated value if it + /// existed or `null` otherwise. + public func get(shm: StableHashMap, k:K) : ?V { + let h = Prim.word32ToNat(keyHash(k)); + let m = shm.table.size(); + let v = if (m > 0) { + AssocList.find(shm.table[h % m], k, keyEq) + } else { + null + }; + }; + /// Insert the value `v` at key `k`. Overwrites an existing entry with key `k` public func put(m: StableHashMap, k : K, v : V) = ignore replace(m, k, v); diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 1f0eaf00cc..bd6f197d53 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -73,6 +73,12 @@ shared ({caller = creator}) actor class () { }; func getAssetEncoding(asset : Asset, acceptEncodings : [Text]) : ?AssetEncoding { + for (acceptEncoding in acceptEncodings.vals()) { + switch (encodings_manipulator.get(asset.encodings, acceptEncoding)) { + case null {}; + case (?assetEncoding) return ?assetEncoding; + } + }; null }; @@ -306,7 +312,6 @@ shared ({caller = creator}) actor class () { }; encodings_manipulator.put(asset.encodings, arg.content_encoding, encoding); - throw Error.reject("set_asset_content: not implemented"); }; }; }; @@ -322,4 +327,7 @@ shared ({caller = creator}) actor class () { public func clear(op: ClearOperation) : async () { throw Error.reject("clear: not implemented"); }; + + public func version_4() : async() { + } }; From 8817568f6f052be4a8444c2f499f1cc4b96895db Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 19 Feb 2021 14:41:33 -0800 Subject: [PATCH 12/48] wip --- src/distributed/assetstorage.mo | 49 ++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index bd6f197d53..8af07a9416 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -18,6 +18,7 @@ shared ({caller = creator}) actor class () { public type BatchId = Nat; public type BlobId = Text; + public type EncodingId = Text; public type Key = Text; public type Path = Text; public type Commit = Bool; @@ -131,6 +132,14 @@ shared ({caller = creator}) actor class () { Int.toText(result) }; + var nextEncodingId = 1; + let encodings = H.HashMap(7, Text.equal, Text.hash); + func allocEncodingId() : EncodingId { + let result = nextEncodingId; + nextEncodingId += 1; + Int.toText(result) + }; + func takeBlob(blobId: BlobId) : ?[Nat8] { switch (blobs.remove(blobId)) { case null null; @@ -142,9 +151,13 @@ shared ({caller = creator}) actor class () { } }; + type Batch = { + expiry : Time; + }; + // We track when each group of blobs should expire, // so that they don't consume space after an interrupted install. - let BATCH_EXPIRY_NANOS = 5 * 1000 * 1000; + let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000; var next_batch_id = 1; type Time = Int; let batch_expiry = H.HashMap(7, Int.equal, Int.hash); @@ -248,6 +261,14 @@ shared ({caller = creator}) actor class () { #ok(blobId) }; + func createEncoding(batch: Batch, chunks: Nat32) : Result.Result { + let encodingId = allocEncodingId(); + let chunkArray = Array.init(Nat32.toNat(chunks), null); + encodings.put(encodingId, chunkArray); + + #ok(encodingId) + }; + public func create_blobs( arg: { blob_info: [ { length: Nat32 } ] } ) : async ( { blob_ids: [BlobId] } ) { @@ -263,6 +284,21 @@ shared ({caller = creator}) actor class () { } }; + public func create_encodings( arg: { + encoding_info: [ { chunks: Nat32 } ] + } ) : async ( { encoding_ids: [EncodingId] } ) { + let batch : Batch = { + expiry = Time.now() + BATCH_EXPIRY_NANOS; + }; + let createEncodingInBatch = func (arg: { chunks: Nat32 }) : Result.Result { + createEncoding(batch, arg.chunks) + }; + switch (Array.mapResult<{chunks: Nat32}, EncodingId, Text>(arg.encoding_info, createEncodingInBatch)) { + case (#ok(ids)) { { encoding_ids = ids } }; + case (#err(err)) throw Error.reject(err); + } + }; + public func write_blob( arg: { blob_id: BlobId; offset: Nat32; @@ -279,6 +315,17 @@ shared ({caller = creator}) actor class () { } }; + public func set_encoding_chunk( arg: { + encoding_id: EncodingId; + index: Nat; + contents: Blob; + } ) : async () { + switch (encodings.get(arg.encoding_id)) { + case null throw Error.reject("Encoding not found"); + case (?encodingChunks) encodingChunks[arg.index] := ?arg.contents; + } + }; + public func batch(ops: [BatchOperationKind]) : async () { throw Error.reject("batch: not implemented"); }; From 75d756137a8a311ef8b3fde472158f34f0ca70b7 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 19 Feb 2021 15:17:57 -0800 Subject: [PATCH 13/48] wip --- src/distributed/assetstorage.mo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 8af07a9416..97167925d9 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -375,6 +375,6 @@ shared ({caller = creator}) actor class () { throw Error.reject("clear: not implemented"); }; - public func version_4() : async() { + public func version_5() : async() { } }; From cf20411ddd393b1ffbddc9f94eaf93c149eceef6 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 19 Feb 2021 18:04:02 -0800 Subject: [PATCH 14/48] first pass chunked retrieval create_batch, create_chunk, create_asset, set_asset_content, get, get_chunk work manually --- src/distributed/assetstorage.mo | 220 +++++++++++++++++++++----------- 1 file changed, 148 insertions(+), 72 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 97167925d9..30ee18544b 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -18,6 +18,7 @@ shared ({caller = creator}) actor class () { public type BatchId = Nat; public type BlobId = Text; + public type ChunkId = Nat; public type EncodingId = Text; public type Key = Text; public type Path = Text; @@ -29,32 +30,33 @@ shared ({caller = creator}) actor class () { public type TotalLength = Nat; - public type CreateAssetOperation = { + public type CreateAssetArguments = { key: Key; content_type: Text; }; - public type SetAssetContentOperation = { + public type SetAssetContentArguments = { key: Key; content_encoding: Text; - blob_id: BlobId; + chunk_ids: [ChunkId] }; - public type UnsetAssetContentOperation = { + public type UnsetAssetContentArguments = { key: Key; content_encoding: Text; }; - public type DeleteAssetOperation = { + public type DeleteAssetArguments = { key: Key; }; - public type ClearOperation = { + public type ClearArguments = { }; public type BatchOperationKind = { - #create: CreateAssetOperation; - #set_content: SetAssetContentOperation; - #unset_content: UnsetAssetContentOperation; + #create_asset: CreateAssetArguments; + #set_asset_content: SetAssetContentArguments; + #unset_asset_content: UnsetAssetContentArguments; - #delete: DeleteAssetOperation; - #clear: ClearOperation; + #delete_asset: DeleteAssetArguments; + + #clear: ClearArguments; }; @@ -65,7 +67,8 @@ shared ({caller = creator}) actor class () { type AssetEncoding = { contentEncoding: Text; - content: [Nat8]; + content: [Blob]; + totalLength: Nat; }; type Asset = { @@ -77,24 +80,25 @@ shared ({caller = creator}) actor class () { for (acceptEncoding in acceptEncodings.vals()) { switch (encodings_manipulator.get(asset.encodings, acceptEncoding)) { case null {}; - case (?assetEncoding) return ?assetEncoding; + case (?encodings) return ?encodings; } }; null }; - stable var asset_entries : [(Key, Asset)] = []; - let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); - let assets_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); - let encodings_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); - system func preupgrade() { - asset_entries := Iter.toArray(assets.entries()); - }; + stable var asset_entries : [(Key, Asset)] = []; + let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); + let assets_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); + let encodings_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); - system func postupgrade() { - asset_entries := []; - }; + system func preupgrade() { + asset_entries := Iter.toArray(assets.entries()); + }; + + system func postupgrade() { + asset_entries := []; + }; // blob data doesn't need to be stable class BlobBuffer(initBatchId: Nat, initBuffer: [var Nat8]) { @@ -123,15 +127,34 @@ shared ({caller = creator}) actor class () { }; }; + type Chunk = { + batch: Batch; + content: Blob; + }; + var nextBlobId = 1; let blobs = H.HashMap(7, Text.equal, Text.hash); + var nextChunkId = 1; + let chunks = H.HashMap(7, Int.equal, Int.hash); + func allocBlobId() : BlobId { let result = nextBlobId; nextBlobId += 1; Int.toText(result) }; + func createChunk(batch: Batch, content: Blob) : ChunkId { + let chunkId = nextChunkId; + nextChunkId += 1; + let chunk : Chunk = { + batch = batch; + content = content; + }; + chunks.put(chunkId, chunk); + chunkId + }; + var nextEncodingId = 1; let encodings = H.HashMap(7, Text.equal, Text.hash); func allocEncodingId() : EncodingId { @@ -151,6 +174,13 @@ shared ({caller = creator}) actor class () { } }; + func takeChunk(chunkId: ChunkId): Result.Result { + switch (chunks.remove(chunkId)) { + case null #err("chunk not found"); + case (?chunk) #ok(chunk.content); + } + }; + type Batch = { expiry : Time; }; @@ -160,12 +190,15 @@ shared ({caller = creator}) actor class () { let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000; var next_batch_id = 1; type Time = Int; - let batch_expiry = H.HashMap(7, Int.equal, Int.hash); + let batches = H.HashMap(7, Int.equal, Int.hash); func startBatch(): BatchId { let batch_id = next_batch_id; next_batch_id += 1; - let expires = Time.now() + BATCH_EXPIRY_NANOS; + let batch : Batch = { + expiry = Time.now() + BATCH_EXPIRY_NANOS; + }; + batches.put(batch_id, batch); batch_id }; @@ -185,22 +218,14 @@ shared ({caller = creator}) actor class () { }; }; - //public query func retrieveX(path : Path) : async Contents { - // let arg = { - // key = path; - // accept_encodings = []; - // }; - // let x = get(arg); - // x.contents - //}; - public query func retrieve(path : Path) : async [Nat8] { + public query func retrieve(path : Path) : async Blob { switch (assets.get(path)) { case null throw Error.reject("not found"); case (?asset) { - switch (getAssetEncoding(asset, [])) { + switch (getAssetEncoding(asset, ["identity"])) { case null throw Error.reject("no such encoding"); case (?encoding) { - encoding.content + encoding.content[0] } }; }; @@ -223,32 +248,51 @@ shared ({caller = creator}) actor class () { public query func get(arg:{ key: Key; accept_encodings: [Text] - }) : async ( { contents: [Nat8]; content_type: Text; content_encoding: Text } ) { + }) : async ( { + content: Blob; + content_type: Text; + content_encoding: Text; + total_length: Nat; + } ) { switch (assets.get(arg.key)) { - case null throw Error.reject("not found"); - case (?asset) { - switch (getAssetEncoding(asset, arg.accept_encodings)) { - case null throw Error.reject("no such encoding"); - case (?encoding) { - { - contents = encoding.content; - content_type = asset.contentType; - content_encoding = encoding.contentEncoding; + case null throw Error.reject("not found"); + case (?asset) { + switch (getAssetEncoding(asset, arg.accept_encodings)) { + case null throw Error.reject("no such encoding"); + case (?encoding) { + { + content = encoding.content[0]; + content_type = asset.contentType; + content_encoding = encoding.contentEncoding; + total_length = encoding.totalLength; + } } - } - }; + }; }; }; }; - //func arrayToBlob(a : [Word8]) : Blob { - // a - //}; - - - //func nat8ArrayToBlob(a : [Nat8]) : Blob { - // a - //}; + public query func get_chunk(arg:{ + key: Key; + content_encoding: Text; + index: Nat; + }) : async ( { + content: Blob + }) { + switch (assets.get(arg.key)) { + case null throw Error.reject("not found"); + case (?asset) { + switch (encodings_manipulator.get(asset.encodings, arg.content_encoding)) { + case null throw Error.reject("no such encoding"); + case (?encoding) { + { + content = encoding.content[arg.index]; + } + } + }; + }; + }; + }; func createBlob(batchId: BatchId, length: Nat32) : Result.Result { let blobId = allocBlobId(); @@ -326,11 +370,35 @@ shared ({caller = creator}) actor class () { } }; + public func create_batch() : async ({ + batch_id: BatchId + }) { + { + batch_id = startBatch(); + } + }; + + public func create_chunk( arg: { + batch_id: BatchId; + content: Blob; + } ) : async ({ + chunk_id: ChunkId + }) { + switch (batches.get(arg.batch_id)) { + case null throw Error.reject("batch not found"); + case (?batch) { + { + chunk_id = createChunk(batch, arg.content) + } + } + } + }; + public func batch(ops: [BatchOperationKind]) : async () { throw Error.reject("batch: not implemented"); }; - public func create_asset(arg: CreateAssetOperation) : async () { + public func create_asset(arg: CreateAssetArguments) : async () { switch (assets.get(arg.key)) { case null { let asset : Asset = { @@ -347,34 +415,42 @@ shared ({caller = creator}) actor class () { } }; - public func set_asset_content(arg: SetAssetContentOperation) : async () { - switch (assets.get(arg.key), takeBlob(arg.blob_id)) { - case (null,null) throw Error.reject("Asset and Blob not found"); - case (null,?blob) throw Error.reject("Asset not found"); - case (?asset,null) throw Error.reject("Blob not found"); - case (?asset,?blob) { - let encoding : AssetEncoding = { - contentEncoding = arg.content_encoding; - content = blob; - }; + func addBlobLength(acc: Nat, blob: Blob): Nat { + acc + blob.size() + }; - encodings_manipulator.put(asset.encodings, arg.content_encoding, encoding); + public func set_asset_content(arg: SetAssetContentArguments) : async () { + switch (assets.get(arg.key)) { + case null throw Error.reject("Asset not found"); + case (?asset) { + switch (Array.mapResult(arg.chunk_ids, takeChunk)) { + case (#ok(chunks)) { + let encoding : AssetEncoding = { + contentEncoding = arg.content_encoding; + content = chunks; + totalLength = Array.foldLeft(chunks, 0, addBlobLength); + }; + + encodings_manipulator.put(asset.encodings, arg.content_encoding, encoding); + }; + case (#err(err)) throw Error.reject(err); + }; }; }; }; - public func unset_asset_content(op: UnsetAssetContentOperation) : async () { + public func unset_asset_content(op: UnsetAssetContentArguments) : async () { throw Error.reject("unset_asset_content: not implemented"); }; - public func delete_asset(op: DeleteAssetOperation) : async () { + public func delete_asset(op: DeleteAssetArguments) : async () { throw Error.reject("delete_asset: not implemented"); }; - public func clear(op: ClearOperation) : async () { + public func clear(op: ClearArguments) : async () { throw Error.reject("clear: not implemented"); }; - public func version_5() : async() { + public func version_8() : async() { } }; From 007260f6afb8bfe7e4d7e68c0ce0474e3da19c4d Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 22 Feb 2021 14:35:31 -0800 Subject: [PATCH 15/48] check write access; move set_asset_contents to non-async function --- src/distributed/assetstorage.mo | 132 ++++++++++++-------------------- 1 file changed, 51 insertions(+), 81 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 30ee18544b..8d49389e75 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -238,12 +238,10 @@ shared ({caller = creator}) actor class () { }; func isSafe(caller: Principal) : Bool { + return true; func eq(value: Principal): Bool = value == caller; Array.find(authorized, eq) != null }; - public query func get2() : async( { contents: Blob } ) { - throw Error.reject("nyi"); - }; public query func get(arg:{ key: Key; @@ -313,77 +311,26 @@ shared ({caller = creator}) actor class () { #ok(encodingId) }; - public func create_blobs( arg: { - blob_info: [ { length: Nat32 } ] - } ) : async ( { blob_ids: [BlobId] } ) { - let batchId = startBatch(); - - let createBlobInBatch = func (arg: { length: Nat32 }) : Result.Result { - createBlob(batchId, arg.length) - }; - - switch (Array.mapResult<{length: Nat32}, BlobId, Text>(arg.blob_info, createBlobInBatch)) { - case (#ok(ids)) { { blob_ids = ids } }; - case (#err(err)) throw Error.reject(err); - } - }; - - public func create_encodings( arg: { - encoding_info: [ { chunks: Nat32 } ] - } ) : async ( { encoding_ids: [EncodingId] } ) { - let batch : Batch = { - expiry = Time.now() + BATCH_EXPIRY_NANOS; - }; - let createEncodingInBatch = func (arg: { chunks: Nat32 }) : Result.Result { - createEncoding(batch, arg.chunks) - }; - switch (Array.mapResult<{chunks: Nat32}, EncodingId, Text>(arg.encoding_info, createEncodingInBatch)) { - case (#ok(ids)) { { encoding_ids = ids } }; - case (#err(err)) throw Error.reject(err); - } - }; - - public func write_blob( arg: { - blob_id: BlobId; - offset: Nat32; - contents: Blob - } ) : async () { - switch (blobs.get(arg.blob_id)) { - case null throw Error.reject("Blob not found"); - case (?blobBuffer) { - switch (blobBuffer.setData(arg.offset, arg.contents)) { - case (#ok) {}; - case (#err(text)) throw Error.reject(text); - } - }; - } - }; - - public func set_encoding_chunk( arg: { - encoding_id: EncodingId; - index: Nat; - contents: Blob; - } ) : async () { - switch (encodings.get(arg.encoding_id)) { - case null throw Error.reject("Encoding not found"); - case (?encodingChunks) encodingChunks[arg.index] := ?arg.contents; - } - }; - - public func create_batch() : async ({ + public shared ({ caller }) func create_batch(arg: {}) : async ({ batch_id: BatchId }) { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); + { batch_id = startBatch(); } }; - public func create_chunk( arg: { + public shared ({ caller }) func create_chunk( arg: { batch_id: BatchId; content: Blob; } ) : async ({ chunk_id: ChunkId }) { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); + switch (batches.get(arg.batch_id)) { case null throw Error.reject("batch not found"); case (?batch) { @@ -394,11 +341,17 @@ shared ({caller = creator}) actor class () { } }; - public func batch(ops: [BatchOperationKind]) : async () { - throw Error.reject("batch: not implemented"); - }; + public shared ({ caller }) func commit_batch(ops: [BatchOperationKind]) : async () { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); + + throw Error.reject("batch: not implemented"); + }; + + public shared ({ caller }) func create_asset(arg: CreateAssetArguments) : async () { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); - public func create_asset(arg: CreateAssetArguments) : async () { switch (assets.get(arg.key)) { case null { let asset : Asset = { @@ -419,9 +372,16 @@ shared ({caller = creator}) actor class () { acc + blob.size() }; - public func set_asset_content(arg: SetAssetContentArguments) : async () { + public shared ({ caller }) func set_asset_content(arg: SetAssetContentArguments) : async () { + switch(do_set_asset_content(caller, arg)) { + case (#ok(())) {}; + case (#err(err)) throw Error.reject(err); + }; + }; + + func do_set_asset_content(caller: Principal, arg: SetAssetContentArguments) : Result.Result<(), Text> { switch (assets.get(arg.key)) { - case null throw Error.reject("Asset not found"); + case null #err("asset not found"); case (?asset) { switch (Array.mapResult(arg.chunk_ids, takeChunk)) { case (#ok(chunks)) { @@ -432,25 +392,35 @@ shared ({caller = creator}) actor class () { }; encodings_manipulator.put(asset.encodings, arg.content_encoding, encoding); + #ok(()); }; - case (#err(err)) throw Error.reject(err); + case (#err(err)) #err(err); }; }; - }; + } }; - public func unset_asset_content(op: UnsetAssetContentArguments) : async () { - throw Error.reject("unset_asset_content: not implemented"); - }; + public shared ({ caller }) func unset_asset_content(op: UnsetAssetContentArguments) : async () { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); - public func delete_asset(op: DeleteAssetArguments) : async () { - throw Error.reject("delete_asset: not implemented"); - }; + throw Error.reject("unset_asset_content: not implemented"); + }; - public func clear(op: ClearArguments) : async () { - throw Error.reject("clear: not implemented"); - }; + public shared ({ caller }) func delete_asset(op: DeleteAssetArguments) : async () { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); - public func version_8() : async() { - } + throw Error.reject("delete_asset: not implemented"); + }; + + public shared ({ caller }) func clear(op: ClearArguments) : async () { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); + + throw Error.reject("clear: not implemented"); + }; + + public func version_9() : async() { + } }; From da9bc177d65e5ee955af07a2f358bf71f01423ef Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 22 Feb 2021 16:56:52 -0800 Subject: [PATCH 16/48] StableHashMap --- src/distributed/assetstorage.mo | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 8d49389e75..46d4ecc1b0 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -87,17 +87,18 @@ shared ({caller = creator}) actor class () { }; - stable var asset_entries : [(Key, Asset)] = []; - let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); + //stable var asset_entries : [(Key, Asset)] = []; + //let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); + stable let assets : SHM.StableHashMap = SHM.StableHashMap(); let assets_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); let encodings_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); system func preupgrade() { - asset_entries := Iter.toArray(assets.entries()); + //asset_entries := Iter.toArray(assets.entries()); }; system func postupgrade() { - asset_entries := []; + //asset_entries := []; }; // blob data doesn't need to be stable @@ -219,7 +220,7 @@ shared ({caller = creator}) actor class () { }; public query func retrieve(path : Path) : async Blob { - switch (assets.get(path)) { + switch (assets_manipulator.get(assets, path)) { case null throw Error.reject("not found"); case (?asset) { switch (getAssetEncoding(asset, ["identity"])) { @@ -252,7 +253,7 @@ shared ({caller = creator}) actor class () { content_encoding: Text; total_length: Nat; } ) { - switch (assets.get(arg.key)) { + switch (assets_manipulator.get(assets, arg.key)) { case null throw Error.reject("not found"); case (?asset) { switch (getAssetEncoding(asset, arg.accept_encodings)) { @@ -277,7 +278,7 @@ shared ({caller = creator}) actor class () { }) : async ( { content: Blob }) { - switch (assets.get(arg.key)) { + switch (assets_manipulator.get(assets, arg.key)) { case null throw Error.reject("not found"); case (?asset) { switch (encodings_manipulator.get(asset.encodings, arg.content_encoding)) { @@ -352,13 +353,13 @@ shared ({caller = creator}) actor class () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - switch (assets.get(arg.key)) { + switch (assets_manipulator.get(assets, arg.key)) { case null { let asset : Asset = { contentType = arg.content_type; encodings = SHM.StableHashMap(); }; - assets.put( (arg.key, asset) ); + assets_manipulator.put(assets, arg.key, asset ); }; case (?asset) { if (asset.contentType != arg.content_type) @@ -380,7 +381,7 @@ shared ({caller = creator}) actor class () { }; func do_set_asset_content(caller: Principal, arg: SetAssetContentArguments) : Result.Result<(), Text> { - switch (assets.get(arg.key)) { + switch (assets_manipulator.get(assets, arg.key)) { case null #err("asset not found"); case (?asset) { switch (Array.mapResult(arg.chunk_ids, takeChunk)) { @@ -421,6 +422,6 @@ shared ({caller = creator}) actor class () { throw Error.reject("clear: not implemented"); }; - public func version_9() : async() { + public func version_10() : async() { } }; From 0a1be8f5c1969d7f0437d52265e26c45a536f9f5 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 22 Feb 2021 18:12:45 -0800 Subject: [PATCH 17/48] split up for commit_batch; --- src/distributed/assetstorage.mo | 61 ++++++++++++++------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 46d4ecc1b0..0218d38da7 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -221,16 +221,16 @@ shared ({caller = creator}) actor class () { public query func retrieve(path : Path) : async Blob { switch (assets_manipulator.get(assets, path)) { - case null throw Error.reject("not found"); - case (?asset) { - switch (getAssetEncoding(asset, ["identity"])) { - case null throw Error.reject("no such encoding"); - case (?encoding) { - encoding.content[0] - } - }; + case null throw Error.reject("not found"); + case (?asset) { + switch (getAssetEncoding(asset, ["identity"])) { + case null throw Error.reject("no such encoding"); + case (?encoding) { + encoding.content[0] + } + }; }; - }; + } }; public query func list() : async [Path] { @@ -254,7 +254,7 @@ shared ({caller = creator}) actor class () { total_length: Nat; } ) { switch (assets_manipulator.get(assets, arg.key)) { - case null throw Error.reject("not found"); + case null throw Error.reject("asset not found"); case (?asset) { switch (getAssetEncoding(asset, arg.accept_encodings)) { case null throw Error.reject("no such encoding"); @@ -279,7 +279,7 @@ shared ({caller = creator}) actor class () { content: Blob }) { switch (assets_manipulator.get(assets, arg.key)) { - case null throw Error.reject("not found"); + case null throw Error.reject("asset not found"); case (?asset) { switch (encodings_manipulator.get(asset.encodings, arg.content_encoding)) { case null throw Error.reject("no such encoding"); @@ -293,25 +293,6 @@ shared ({caller = creator}) actor class () { }; }; - func createBlob(batchId: BatchId, length: Nat32) : Result.Result { - let blobId = allocBlobId(); - - let blob = Array.init(Nat32.toNat(length), 0); - let blobBuffer = BlobBuffer(batchId, blob); - - blobs.put(blobId, blobBuffer); - - #ok(blobId) - }; - - func createEncoding(batch: Batch, chunks: Nat32) : Result.Result { - let encodingId = allocEncodingId(); - let chunkArray = Array.init(Nat32.toNat(chunks), null); - encodings.put(encodingId, chunkArray); - - #ok(encodingId) - }; - public shared ({ caller }) func create_batch(arg: {}) : async ({ batch_id: BatchId }) { @@ -353,6 +334,13 @@ shared ({caller = creator}) actor class () { if (isSafe(caller) == false) throw Error.reject("not authorized"); + switch(do_create_asset(arg)) { + case (#ok(())) {}; + case (#err(err)) throw Error.reject(err); + }; + }; + + func do_create_asset(arg: CreateAssetArguments) : Result.Result<(), Text> { switch (assets_manipulator.get(assets, arg.key)) { case null { let asset : Asset = { @@ -363,10 +351,10 @@ shared ({caller = creator}) actor class () { }; case (?asset) { if (asset.contentType != arg.content_type) - throw Error.reject("create_asset: content type mismatch"); - + return #err("create_asset: content type mismatch"); } - } + }; + #ok(()) }; func addBlobLength(acc: Nat, blob: Blob): Nat { @@ -374,13 +362,16 @@ shared ({caller = creator}) actor class () { }; public shared ({ caller }) func set_asset_content(arg: SetAssetContentArguments) : async () { - switch(do_set_asset_content(caller, arg)) { + if (isSafe(caller) == false) + throw Error.reject("not authorized"); + + switch(do_set_asset_content(arg)) { case (#ok(())) {}; case (#err(err)) throw Error.reject(err); }; }; - func do_set_asset_content(caller: Principal, arg: SetAssetContentArguments) : Result.Result<(), Text> { + func do_set_asset_content(arg: SetAssetContentArguments) : Result.Result<(), Text> { switch (assets_manipulator.get(assets, arg.key)) { case null #err("asset not found"); case (?asset) { From 3efc7215848d62de9ca8f9eb5cc98f1afd39c3d2 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Tue, 23 Feb 2021 15:15:07 -0800 Subject: [PATCH 18/48] adjust spacing for Motoko style guide, delete dead code, implement delete_asset --- src/distributed/StableHashMap.mo | 58 +++++++++ src/distributed/assetstorage.mo | 210 ++++++++++++++----------------- 2 files changed, 149 insertions(+), 119 deletions(-) diff --git a/src/distributed/StableHashMap.mo b/src/distributed/StableHashMap.mo index 08cf77a2c9..63e30e7311 100644 --- a/src/distributed/StableHashMap.mo +++ b/src/distributed/StableHashMap.mo @@ -21,6 +21,33 @@ public class StableHashMapManipulator( keyHash: K -> Hash.Hash ) { + /// Returns the number of entries in this HashMap. + public func size(shm: StableHashMap) : Nat = shm._count; + + /// Deletes the entry with the key `k`. Doesn't do anything if the key doesn't + /// exist. + public func delete(shm: StableHashMap, k : K) = ignore remove(shm, k); + + /// Removes the entry with the key `k` and returns the associated value if it + /// existed or `null` otherwise. + public func remove(shm: StableHashMap, k : K) : ?V { + let h = Prim.word32ToNat(keyHash(k)); + let m = shm.table.size(); + let pos = h % m; + if (m > 0) { + let (kvs2, ov) = AssocList.replace(shm.table[pos], k, keyEq, null); + shm.table[pos] := kvs2; + switch(ov){ + case null { }; + case _ { shm._count -= 1; } + }; + ov + } else { + null + }; + }; + + /// Gets the entry with the key `k` and returns its associated value if it /// existed or `null` otherwise. public func get(shm: StableHashMap, k:K) : ?V { @@ -79,6 +106,37 @@ public class StableHashMapManipulator( ov }; + /// Returns an iterator over the key value pairs in this + /// HashMap. Does _not_ modify the HashMap. + public func entries(m: StableHashMap) : Iter.Iter<(K,V)> { + if (m.table.size() == 0) { + object { public func next() : ?(K,V) { null } } + } + else { + object { + var kvs = m.table[0]; + var nextTablePos = 1; + public func next () : ?(K,V) { + switch kvs { + case (?(kv, kvs2)) { + kvs := kvs2; + ?kv + }; + case null { + if (nextTablePos < m.table.size()) { + kvs := m.table[nextTablePos]; + nextTablePos += 1; + next() + } else { + null + } + } + } + }; + } + } + }; + }; } \ No newline at end of file diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 0218d38da7..4d8dd7267d 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -16,54 +16,55 @@ import Word8 "mo:base/Word8"; shared ({caller = creator}) actor class () { - public type BatchId = Nat; - public type BlobId = Text; - public type ChunkId = Nat; - public type EncodingId = Text; - public type Key = Text; - public type Path = Text; - public type Commit = Bool; - public type Contents = Blob; - public type ContentEncoding = Text; - public type ContentType = Text; - public type Offset = Nat; - public type TotalLength = Nat; - - - public type CreateAssetArguments = { - key: Key; - content_type: Text; - }; - public type SetAssetContentArguments = { - key: Key; - content_encoding: Text; - chunk_ids: [ChunkId] - }; - public type UnsetAssetContentArguments = { - key: Key; - content_encoding: Text; - }; - public type DeleteAssetArguments = { - key: Key; - }; - public type ClearArguments = { - }; + // old interface: + public type Path = Text; + public type Contents = Blob; + + // new hotness + public type BatchId = Nat; + public type BlobId = Text; + public type ChunkId = Nat; + public type EncodingId = Text; + public type Key = Text; + public type ContentEncoding = Text; + public type ContentType = Text; + public type Offset = Nat; + public type TotalLength = Nat; + + + public type CreateAssetArguments = { + key: Key; + content_type: Text; + }; + public type SetAssetContentArguments = { + key: Key; + content_encoding: Text; + chunk_ids: [ChunkId] + }; + public type UnsetAssetContentArguments = { + key: Key; + content_encoding: Text; + }; + public type DeleteAssetArguments = { + key: Key; + }; + public type ClearArguments = { + }; - public type BatchOperationKind = { - #create_asset: CreateAssetArguments; - #set_asset_content: SetAssetContentArguments; - #unset_asset_content: UnsetAssetContentArguments; + public type BatchOperationKind = { + #create_asset: CreateAssetArguments; + #set_asset_content: SetAssetContentArguments; + #unset_asset_content: UnsetAssetContentArguments; - #delete_asset: DeleteAssetArguments; + #delete_asset: DeleteAssetArguments; - #clear: ClearArguments; - }; + #clear: ClearArguments; + }; - stable var authorized: [Principal] = [creator]; + stable var authorized: [Principal] = [creator]; - let db: Tree.RBTree = Tree.RBTree(Text.compare); type AssetEncoding = { contentEncoding: Text; @@ -101,50 +102,14 @@ shared ({caller = creator}) actor class () { //asset_entries := []; }; - // blob data doesn't need to be stable - class BlobBuffer(initBatchId: Nat, initBuffer: [var Nat8]) { - let batchId = initBatchId; - let buffer = initBuffer; - - public func setData(offset: Nat32, data: Blob): Result.Result<(), Text> { - var index: Nat = Nat32.toNat(offset); - - if (index + data.size() > buffer.size()) { - #err("overflow: offset " # Nat32.toText(offset) # - " + data size " # Nat.toText(data.size()) # - " exceeds blob size of " # Nat.toText(buffer.size())) - } else { - for (b in data.bytes()) { - buffer[index] := Nat8.fromNat(Word8.toNat(b)); - index += 1; - }; - #ok() - } - }; - - public func takeBuffer() : [Nat8] { - let x = Array.freeze(buffer); - x - }; - }; - type Chunk = { batch: Batch; content: Blob; }; - var nextBlobId = 1; - let blobs = H.HashMap(7, Text.equal, Text.hash); - var nextChunkId = 1; let chunks = H.HashMap(7, Int.equal, Int.hash); - func allocBlobId() : BlobId { - let result = nextBlobId; - nextBlobId += 1; - Int.toText(result) - }; - func createChunk(batch: Batch, content: Blob) : ChunkId { let chunkId = nextChunkId; nextChunkId += 1; @@ -156,24 +121,8 @@ shared ({caller = creator}) actor class () { chunkId }; - var nextEncodingId = 1; - let encodings = H.HashMap(7, Text.equal, Text.hash); - func allocEncodingId() : EncodingId { - let result = nextEncodingId; - nextEncodingId += 1; - Int.toText(result) - }; - - func takeBlob(blobId: BlobId) : ?[Nat8] { - switch (blobs.remove(blobId)) { - case null null; - case (?blobBuffer) { - let b: [Nat8] = blobBuffer.takeBuffer(); - //let blob : Blob = b; - ?b - } - } - }; + //var nextEncodingId = 1; + //let encodings = H.HashMap(7, Text.equal, Text.hash); func takeChunk(chunkId: ChunkId): Result.Result { switch (chunks.remove(chunkId)) { @@ -203,22 +152,45 @@ shared ({caller = creator}) actor class () { batch_id }; - public shared ({ caller }) func authorize(other: Principal) : async () { - if (isSafe(caller)) { - authorized := Array.append(authorized, [other]); - } else { - throw Error.reject("not authorized"); - } + public shared ({ caller }) func authorize(other: Principal) : async () { + if (isSafe(caller)) { + authorized := Array.append(authorized, [other]); + } else { + throw Error.reject("not authorized"); + } + }; + + public shared ({ caller }) func store(path : Path, contents : Contents) : async () { + if (isSafe(caller) == false) { + throw Error.reject("not authorized"); }; - public shared ({ caller }) func store(path : Path, contents : Contents) : async () { - if (isSafe(caller)) { - db.put(path, contents); - } else { - throw Error.reject("not authorized"); - }; + let batch_id = startBatch(); + let chunk_id = switch (batches.get(batch_id)) { + case null throw Error.reject("batch not found"); + case (?batch) createChunk(batch, contents) }; + let create_asset_args : CreateAssetArguments = { + key = path; + content_type = "application/octet-stream" + }; + switch(do_create_asset(create_asset_args)) { + case (#ok(())) {}; + case (#err(msg)) throw Error.reject(msg); + }; + + let set_asset_content_args : SetAssetContentArguments = { + key = path; + content_encoding = "identity"; + chunk_ids = [ chunk_id ]; + }; + switch(do_set_asset_content(set_asset_content_args)) { + case (#ok(())) {}; + case (#err(msg)) throw Error.reject(msg); + }; + }; + public query func retrieve(path : Path) : async Blob { switch (assets_manipulator.get(assets, path)) { case null throw Error.reject("not found"); @@ -233,16 +205,16 @@ shared ({caller = creator}) actor class () { } }; - public query func list() : async [Path] { - let iter = Iter.map<(Path, Contents), Path>(db.entries(), func (path, _) = path); - Iter.toArray(iter) - }; + public query func list() : async [Path] { + let iter = Iter.map<(Text, Asset), Path>(assets_manipulator.entries(assets), func (key, _) = key); + Iter.toArray(iter) + }; - func isSafe(caller: Principal) : Bool { - return true; - func eq(value: Principal): Bool = value == caller; - Array.find(authorized, eq) != null - }; + func isSafe(caller: Principal) : Bool { + //return true; + func eq(value: Principal): Bool = value == caller; + Array.find(authorized, eq) != null + }; public query func get(arg:{ key: Key; @@ -311,7 +283,7 @@ shared ({caller = creator}) actor class () { chunk_id: ChunkId }) { if (isSafe(caller) == false) - throw Error.reject("not authorized"); + throw Error.reject("not authorized"); switch (batches.get(arg.batch_id)) { case null throw Error.reject("batch not found"); @@ -403,7 +375,7 @@ shared ({caller = creator}) actor class () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - throw Error.reject("delete_asset: not implemented"); + assets_manipulator.delete(assets, op.key); }; public shared ({ caller }) func clear(op: ClearArguments) : async () { @@ -413,6 +385,6 @@ shared ({caller = creator}) actor class () { throw Error.reject("clear: not implemented"); }; - public func version_10() : async() { + public func version_11() : async() { } }; From 5d4d11736ff3243f839b5f82cba304aca394a195 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 24 Feb 2021 09:25:10 -0800 Subject: [PATCH 19/48] wip --- src/dfx/src/lib/installers/assets.rs | 94 ++++++++++++++++++++++++++-- src/distributed/assetstorage.mo | 2 +- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index d4aa060f0e..9b266d5db9 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -2,12 +2,67 @@ use crate::lib::canister_info::assets::AssetsCanisterInfo; use crate::lib::canister_info::CanisterInfo; use crate::lib::error::DfxResult; use crate::lib::waiter::waiter_with_timeout; -use candid::Encode; +use candid::{CandidType, Decode, Encode}; use ic_agent::Agent; -use std::path::Path; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; use std::time::Duration; -use walkdir::WalkDir; +use walkdir::{DirEntry, WalkDir}; + +const GET: &str = "get"; +const CREATE_BATCH: &str = "create_batch"; + +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct CreateBatchRequest {} + +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct CreateBatchResponse { + batch_id: u128, +} + +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct CreateChunkRequest { + batch_id: u128, + content: Vec, +} + +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct CreateChunkResponse { + chunk_id: u128, +} + +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct GetRequest { + key: String, + accept_encodings: Vec, +} + +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct GetResponse { + contents: Vec, + content_type: String, + content_encoding: String, + +} + +#[derive(Clone, Debug)] +struct AssetLocation { + source: PathBuf, + relative: PathBuf, +} + +struct ChunkedAsset { + asset_location: AssetLocation, + chunk_ids: Vec, +} + +fn make_chunked_asset(agent: &Agent, asset_location: AssetLocation) -> DfxResult { + Ok(ChunkedAsset { + asset_location: asset_location, + chunk_ids: vec![], + }) +} pub async fn post_install_store_assets( info: &CanisterInfo, @@ -17,6 +72,38 @@ pub async fn post_install_store_assets( let assets_canister_info = info.as_info::()?; let output_assets_path = assets_canister_info.get_output_assets_path(); + let asset_locations: Vec = WalkDir::new(output_assets_path) + .into_iter() + .filter_map(|r| { + r.ok().filter(|entry| entry.file_type().is_file()).map(|e| { + let source = e.path().to_path_buf(); + let relative = source + .strip_prefix(output_assets_path) + .expect("cannot strip prefix") + .to_path_buf(); + AssetLocation { source, relative } + }) + }) + .collect(); + + let canister_id = info.get_canister_id().expect("Could not find canister ID."); + + let create_batch_args = CreateBatchRequest {}; + let response = agent + .update(&canister_id, CREATE_BATCH) + .with_arg(candid::Encode!(&create_batch_args)?) + .expire_after(timeout) + .call_and_wait(waiter_with_timeout(timeout)) + .await?; + let create_batch_response = candid::Decode!(&response, CreateBatchResponse)?; + let batch_id = create_batch_response.batch_id; + + let chunked_assets: DfxResult> = asset_locations + .into_iter() + .map(|loc| make_chunked_asset(agent, loc)) + .collect(); + let chunked_assets = chunked_assets?; + let walker = WalkDir::new(output_assets_path).into_iter(); for entry in walker { let entry = entry?; @@ -29,7 +116,6 @@ pub async fn post_install_store_assets( let path = relative.to_string_lossy().to_string(); let blob = candid::Encode!(&path, &content)?; - let canister_id = info.get_canister_id().expect("Could not find canister ID."); let method_name = String::from("store"); agent diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 4d8dd7267d..23d6876671 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -385,6 +385,6 @@ shared ({caller = creator}) actor class () { throw Error.reject("clear: not implemented"); }; - public func version_11() : async() { + public func version_12() : async() { } }; From f8cd7fc0d891bdf5a63b8dccc5e379b6e310b0ae Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 24 Feb 2021 16:19:03 -0800 Subject: [PATCH 20/48] StableHashMap: fix divide-by-zero in remove --- src/distributed/StableHashMap.mo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/distributed/StableHashMap.mo b/src/distributed/StableHashMap.mo index 63e30e7311..81d8990bc2 100644 --- a/src/distributed/StableHashMap.mo +++ b/src/distributed/StableHashMap.mo @@ -33,8 +33,8 @@ public class StableHashMapManipulator( public func remove(shm: StableHashMap, k : K) : ?V { let h = Prim.word32ToNat(keyHash(k)); let m = shm.table.size(); - let pos = h % m; if (m > 0) { + let pos = h % m; let (kvs2, ov) = AssocList.replace(shm.table[pos], k, keyEq, null); shm.table[pos] := kvs2; switch(ov){ From a32e0e5110cf1c9149e68766ac2798492cca8fd0 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 24 Feb 2021 16:21:01 -0800 Subject: [PATCH 21/48] CANISTER: fix variant names, commit_batch parameter --- src/distributed/assetstorage.mo | 96 ++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 24 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 23d6876671..6509555622 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -1,5 +1,6 @@ import Array "mo:base/Array"; import Buffer "mo:base/Buffer"; +import Debug "mo:base/Debug"; import Error "mo:base/Error"; import H "mo:base/HashMap"; import Int "mo:base/Int"; @@ -52,16 +53,20 @@ shared ({caller = creator}) actor class () { }; public type BatchOperationKind = { - #create_asset: CreateAssetArguments; - #set_asset_content: SetAssetContentArguments; - #unset_asset_content: UnsetAssetContentArguments; + #CreateAsset: CreateAssetArguments; + #SetAssetContent: SetAssetContentArguments; + #UnsetAssetContent: UnsetAssetContentArguments; - #delete_asset: DeleteAssetArguments; + #DeleteAsset: DeleteAssetArguments; - #clear: ClearArguments; + #Clear: ClearArguments; }; + public type CommitBatchArguments = { + batch_id: BatchId; + operations: [BatchOperationKind]; + }; stable var authorized: [Principal] = [creator]; @@ -175,7 +180,7 @@ shared ({caller = creator}) actor class () { key = path; content_type = "application/octet-stream" }; - switch(do_create_asset(create_asset_args)) { + switch(createAsset(create_asset_args)) { case (#ok(())) {}; case (#err(msg)) throw Error.reject(msg); }; @@ -185,7 +190,7 @@ shared ({caller = creator}) actor class () { content_encoding = "identity"; chunk_ids = [ chunk_id ]; }; - switch(do_set_asset_content(set_asset_content_args)) { + switch(setAssetContent(set_asset_content_args)) { case (#ok(())) {}; case (#err(msg)) throw Error.reject(msg); }; @@ -295,24 +300,42 @@ shared ({caller = creator}) actor class () { } }; - public shared ({ caller }) func commit_batch(ops: [BatchOperationKind]) : async () { + public shared ({ caller }) func commit_batch(args: CommitBatchArguments) : async () { + Debug.print("commit_batch (1)"); if (isSafe(caller) == false) throw Error.reject("not authorized"); - throw Error.reject("batch: not implemented"); + Debug.print("commit_batch (2)"); + for (op in args.operations.vals()) { + Debug.print("commit_batch (3)"); + + let r : Result.Result<(), Text> = switch(op) { + case (#CreateAsset(args)) { createAsset(args); }; + case (#SetAssetContent(args)) { setAssetContent(args); }; + case (#UnsetAssetContent(args)) { unsetAssetContent(args); }; + case (#DeleteAsset(args)) { deleteAsset(args); }; + case (#Clear(args)) { clearEverything(args); } + }; + Debug.print("commit_batch (4)"); + switch(r) { + case (#ok(())) {}; + case (#err(msg)) throw Error.reject(msg); + }; + } }; public shared ({ caller }) func create_asset(arg: CreateAssetArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - switch(do_create_asset(arg)) { + switch(createAsset(arg)) { case (#ok(())) {}; case (#err(err)) throw Error.reject(err); }; }; - func do_create_asset(arg: CreateAssetArguments) : Result.Result<(), Text> { + func createAsset(arg: CreateAssetArguments) : Result.Result<(), Text> { + Debug.print("createAsset()"); switch (assets_manipulator.get(assets, arg.key)) { case null { let asset : Asset = { @@ -329,21 +352,22 @@ shared ({caller = creator}) actor class () { #ok(()) }; - func addBlobLength(acc: Nat, blob: Blob): Nat { - acc + blob.size() - }; - public shared ({ caller }) func set_asset_content(arg: SetAssetContentArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - switch(do_set_asset_content(arg)) { + switch(setAssetContent(arg)) { case (#ok(())) {}; case (#err(err)) throw Error.reject(err); }; }; - func do_set_asset_content(arg: SetAssetContentArguments) : Result.Result<(), Text> { + func addBlobLength(acc: Nat, blob: Blob): Nat { + acc + blob.size() + }; + + func setAssetContent(arg: SetAssetContentArguments) : Result.Result<(), Text> { + Debug.print("setAssetContent()"); switch (assets_manipulator.get(assets, arg.key)) { case null #err("asset not found"); case (?asset) { @@ -364,27 +388,51 @@ shared ({caller = creator}) actor class () { } }; - public shared ({ caller }) func unset_asset_content(op: UnsetAssetContentArguments) : async () { + public shared ({ caller }) func unset_asset_content(args: UnsetAssetContentArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - throw Error.reject("unset_asset_content: not implemented"); + switch(unsetAssetContent(args)) { + case (#ok(())) {}; + case (#err(err)) throw Error.reject(err); + }; + }; + + func unsetAssetContent(args: UnsetAssetContentArguments) : Result.Result<(), Text> { + #err("unset_asset_content: not implemented"); }; - public shared ({ caller }) func delete_asset(op: DeleteAssetArguments) : async () { + public shared ({ caller }) func delete_asset(args: DeleteAssetArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - assets_manipulator.delete(assets, op.key); + switch(deleteAsset(args)) { + case (#ok(())) {}; + case (#err(err)) throw Error.reject(err); + }; }; - public shared ({ caller }) func clear(op: ClearArguments) : async () { + func deleteAsset(args: DeleteAssetArguments) : Result.Result<(), Text> { + Debug.print("deleteAsset() enter"); + assets_manipulator.delete(assets, args.key); + Debug.print("deleteAsset() leave"); + #ok(()) + }; + + public shared ({ caller }) func clear(args: ClearArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - throw Error.reject("clear: not implemented"); + switch(clearEverything(args)) { + case (#ok(())) {}; + case (#err(err)) throw Error.reject(err); + }; + }; + + func clearEverything(args: ClearArguments) : Result.Result<(), Text> { + #err("clear: not implemented") }; - public func version_12() : async() { + public func version_13() : async() { } }; From a96425689602bf39098efbeae8fc5b1e1e3ed3f5 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 25 Feb 2021 09:41:38 -0800 Subject: [PATCH 22/48] CANISTER: debug prints --- src/distributed/assetstorage.mo | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 6509555622..3ec2d96d68 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -276,6 +276,8 @@ shared ({caller = creator}) actor class () { if (isSafe(caller) == false) throw Error.reject("not authorized"); + Debug.print("create_batch"); + { batch_id = startBatch(); } @@ -287,6 +289,7 @@ shared ({caller = creator}) actor class () { } ) : async ({ chunk_id: ChunkId }) { + Debug.print("create_chunk(batch " # Int.toText(arg.batch_id) # ", " # Int.toText(arg.content.size()) # " bytes)"); if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -301,13 +304,11 @@ shared ({caller = creator}) actor class () { }; public shared ({ caller }) func commit_batch(args: CommitBatchArguments) : async () { - Debug.print("commit_batch (1)"); + Debug.print("commit_batch (" # Int.toText(args.operations.size()) # ")"); if (isSafe(caller) == false) throw Error.reject("not authorized"); - Debug.print("commit_batch (2)"); for (op in args.operations.vals()) { - Debug.print("commit_batch (3)"); let r : Result.Result<(), Text> = switch(op) { case (#CreateAsset(args)) { createAsset(args); }; @@ -316,7 +317,6 @@ shared ({caller = creator}) actor class () { case (#DeleteAsset(args)) { deleteAsset(args); }; case (#Clear(args)) { clearEverything(args); } }; - Debug.print("commit_batch (4)"); switch(r) { case (#ok(())) {}; case (#err(msg)) throw Error.reject(msg); @@ -335,7 +335,7 @@ shared ({caller = creator}) actor class () { }; func createAsset(arg: CreateAssetArguments) : Result.Result<(), Text> { - Debug.print("createAsset()"); + Debug.print("createAsset(" # arg.key # ")"); switch (assets_manipulator.get(assets, arg.key)) { case null { let asset : Asset = { @@ -367,7 +367,7 @@ shared ({caller = creator}) actor class () { }; func setAssetContent(arg: SetAssetContentArguments) : Result.Result<(), Text> { - Debug.print("setAssetContent()"); + Debug.print("setAssetContent(" # arg.key # ")"); switch (assets_manipulator.get(assets, arg.key)) { case null #err("asset not found"); case (?asset) { @@ -413,9 +413,8 @@ shared ({caller = creator}) actor class () { }; func deleteAsset(args: DeleteAssetArguments) : Result.Result<(), Text> { - Debug.print("deleteAsset() enter"); + Debug.print("deleteAsset(" # args.key # ")"); assets_manipulator.delete(assets, args.key); - Debug.print("deleteAsset() leave"); #ok(()) }; From c5099983665e19cb1f39ec9eabaa633cc5979ea5 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 24 Feb 2021 12:39:41 -0800 Subject: [PATCH 23/48] DFX work: create batch and chunks --- src/dfx/src/lib/installers/assets.rs | 118 +++++++++++++++++++++------ 1 file changed, 94 insertions(+), 24 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 9b266d5db9..62e25f11b8 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -1,10 +1,12 @@ use crate::lib::canister_info::assets::AssetsCanisterInfo; use crate::lib::canister_info::CanisterInfo; -use crate::lib::error::DfxResult; +use crate::lib::error::{DfxError, DfxResult}; use crate::lib::waiter::waiter_with_timeout; use candid::{CandidType, Decode, Encode}; +use futures::future::try_join_all; use ic_agent::Agent; +use ic_types::Principal; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -12,6 +14,8 @@ use walkdir::{DirEntry, WalkDir}; const GET: &str = "get"; const CREATE_BATCH: &str = "create_batch"; +const CREATE_CHUNK: &str = "create_chunk"; +const MAX_CHUNK_SIZE: usize = 1_900_000; #[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] struct CreateBatchRequest {} @@ -22,9 +26,9 @@ struct CreateBatchResponse { } #[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] -struct CreateChunkRequest { +struct CreateChunkRequest<'a> { batch_id: u128, - content: Vec, + content: &'a [u8], } #[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] @@ -43,7 +47,6 @@ struct GetResponse { contents: Vec, content_type: String, content_encoding: String, - } #[derive(Clone, Debug)] @@ -54,14 +57,63 @@ struct AssetLocation { struct ChunkedAsset { asset_location: AssetLocation, - chunk_ids: Vec, + chunk_ids: Vec, } -fn make_chunked_asset(agent: &Agent, asset_location: AssetLocation) -> DfxResult { - Ok(ChunkedAsset { - asset_location: asset_location, - chunk_ids: vec![], - }) +async fn make_chunked_asset( + agent: &Agent, + canister_id: &Principal, + timeout: Duration, + batch_id: u128, + asset_location: AssetLocation, +) -> DfxResult { + let content = &std::fs::read(&asset_location.source)?; + println!( + "create chunks for {}", + asset_location.source.to_string_lossy() + ); + let chunks_futures: Vec<_> = content + .chunks(MAX_CHUNK_SIZE) + .map(|content| async move { + let args = CreateChunkRequest { batch_id, content }; + let args = candid::Encode!(&args).expect("unable to encode create_chunk argument"); + println!("create chunk"); + agent + .update(&canister_id, CREATE_CHUNK) + .with_arg(args) + .expire_after(timeout) + .call_and_wait(waiter_with_timeout(timeout)) + .await + .map_err(DfxError::from) + .and_then(|response| { + candid::Decode!(&response, CreateChunkResponse) + .map_err(DfxError::from) + .map(|x| x.chunk_id) + }) + }) + .collect(); + println!("await chunk creation"); + + try_join_all(chunks_futures) + .await + .map(|chunk_ids| ChunkedAsset { + asset_location, + chunk_ids, + }) +} + +async fn make_chunked_assets( + agent: &Agent, + canister_id: &Principal, + timeout: Duration, + batch_id: u128, + locs: Vec, +) -> DfxResult> { + let futs: Vec<_> = locs + .into_iter() + .map(|loc| async { make_chunked_asset(agent, canister_id, timeout, batch_id, loc).await }) + .collect(); + try_join_all(futs).await } pub async fn post_install_store_assets( @@ -88,21 +140,22 @@ pub async fn post_install_store_assets( let canister_id = info.get_canister_id().expect("Could not find canister ID."); - let create_batch_args = CreateBatchRequest {}; - let response = agent - .update(&canister_id, CREATE_BATCH) - .with_arg(candid::Encode!(&create_batch_args)?) - .expire_after(timeout) - .call_and_wait(waiter_with_timeout(timeout)) - .await?; - let create_batch_response = candid::Decode!(&response, CreateBatchResponse)?; - let batch_id = create_batch_response.batch_id; + let batch_id = create_batch(agent, &canister_id, timeout).await?; - let chunked_assets: DfxResult> = asset_locations - .into_iter() - .map(|loc| make_chunked_asset(agent, loc)) - .collect(); - let chunked_assets = chunked_assets?; + let chunked_assets = + make_chunked_assets(agent, &canister_id, timeout, batch_id, asset_locations).await?; + println!("created all chunks"); + + // let mut futs: Vec<_> = vec![]; + // for loc in asset_locations { + // let y = make_chunked_asset(agent, &canister_id, timeout, batch_id, loc); + // futs.append(y); + // } + // let chunked_asset_futures: Vec<_> = asset_locations + // .into_iter() + // .map(|loc| async { make_chunked_asset(agent, &canister_id, timeout, batch_id, loc) }) + // .collect(); + // let x: Vec = try_join_all(chunked_asset_futures).await.unwrap(); let walker = WalkDir::new(output_assets_path).into_iter(); for entry in walker { @@ -128,3 +181,20 @@ pub async fn post_install_store_assets( } Ok(()) } + +async fn create_batch( + agent: &Agent, + canister_id: &Principal, + timeout: Duration, +) -> DfxResult { + let create_batch_args = CreateBatchRequest {}; + let response = agent + .update(&canister_id, CREATE_BATCH) + .with_arg(candid::Encode!(&create_batch_args)?) + .expire_after(timeout) + .call_and_wait(waiter_with_timeout(timeout)) + .await?; + let create_batch_response = candid::Decode!(&response, CreateBatchResponse)?; + let batch_id = create_batch_response.batch_id; + Ok(batch_id) +} From 1fea9bfbcd58a4b938b69a252af421d0d6a96d23 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 24 Feb 2021 16:21:16 -0800 Subject: [PATCH 24/48] DFX: commit batch --- src/dfx/src/lib/installers/assets.rs | 153 ++++++++++++++++++++------- 1 file changed, 117 insertions(+), 36 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 62e25f11b8..3f7540dd30 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -8,14 +8,15 @@ use futures::future::try_join_all; use ic_agent::Agent; use ic_types::Principal; use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::Duration; -use walkdir::{DirEntry, WalkDir}; +use walkdir::WalkDir; -const GET: &str = "get"; +//const GET: &str = "get"; const CREATE_BATCH: &str = "create_batch"; const CREATE_CHUNK: &str = "create_chunk"; -const MAX_CHUNK_SIZE: usize = 1_900_000; +const COMMIT_BATCH: &str = "commit_batch"; +const MAX_CHUNK_SIZE: usize = 300; #[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] struct CreateBatchRequest {} @@ -49,6 +50,48 @@ struct GetResponse { content_encoding: String, } +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct CreateAssetArguments { + key: String, + content_type: String, +} +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct SetAssetContentArguments { + key: String, + content_encoding: String, + chunk_ids: Vec, +} +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct UnsetAssetContentArguments { + key: String, + content_encoding: String, +} +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct DeleteAssetArguments { + key: String, +} +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct ClearArguments {} + +#[derive(CandidType, Clone, Debug, Serialize, Deserialize)] +enum BatchOperationKind { + CreateAsset(CreateAssetArguments), + + SetAssetContent(SetAssetContentArguments), + + UnsetAssetContent(UnsetAssetContentArguments), + + DeleteAsset(DeleteAssetArguments), + + Clear(ClearArguments), +} + +#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +struct CommitBatchArguments { + batch_id: u128, + operations: Vec, +} + #[derive(Clone, Debug)] struct AssetLocation { source: PathBuf, @@ -116,6 +159,51 @@ async fn make_chunked_assets( try_join_all(futs).await } +async fn commit_batch( + agent: &Agent, + canister_id: &Principal, + timeout: Duration, + batch_id: u128, + chunked_assets: Vec, +) -> DfxResult { + let operations: Vec<_> = chunked_assets + .into_iter() + .map(|chunked_asset| { + vec![ + BatchOperationKind::DeleteAsset(DeleteAssetArguments { + key: chunked_asset.asset_location.relative.to_string_lossy().to_string(), + }), + BatchOperationKind::CreateAsset(CreateAssetArguments { + key: chunked_asset.asset_location.relative.to_string_lossy().to_string(), + content_type: "application/octet-stream".to_string(), + }), + BatchOperationKind::SetAssetContent(SetAssetContentArguments { + key: chunked_asset.asset_location.relative.to_string_lossy().to_string(), + content_encoding: "identity".to_string(), + chunk_ids: chunked_asset.chunk_ids, + }), + ] + }) + .flatten() + .collect(); + let arg = CommitBatchArguments { + batch_id, + operations, + }; + let arg = candid::Encode!(&arg)?; + println!("encoded arg: {:02X?}", arg); + let idl = candid::IDLArgs::from_bytes(&arg)?; + println!("{:?}", idl); + let x = agent + .update(&canister_id, COMMIT_BATCH) + .with_arg(arg) + .expire_after(timeout) + .call_and_wait(waiter_with_timeout(timeout)) + .await; + x?; + Ok(()) +} + pub async fn post_install_store_assets( info: &CanisterInfo, agent: &Agent, @@ -146,39 +234,32 @@ pub async fn post_install_store_assets( make_chunked_assets(agent, &canister_id, timeout, batch_id, asset_locations).await?; println!("created all chunks"); - // let mut futs: Vec<_> = vec![]; - // for loc in asset_locations { - // let y = make_chunked_asset(agent, &canister_id, timeout, batch_id, loc); - // futs.append(y); - // } - // let chunked_asset_futures: Vec<_> = asset_locations - // .into_iter() - // .map(|loc| async { make_chunked_asset(agent, &canister_id, timeout, batch_id, loc) }) - // .collect(); - // let x: Vec = try_join_all(chunked_asset_futures).await.unwrap(); - - let walker = WalkDir::new(output_assets_path).into_iter(); - for entry in walker { - let entry = entry?; - if entry.file_type().is_file() { - let source = entry.path(); - let relative: &Path = source - .strip_prefix(output_assets_path) - .expect("cannot strip prefix"); - let content = &std::fs::read(&source)?; - let path = relative.to_string_lossy().to_string(); - let blob = candid::Encode!(&path, &content)?; - - let method_name = String::from("store"); + println!("committing batch"); + commit_batch(agent, &canister_id, timeout, batch_id, chunked_assets).await?; + println!("committed"); - agent - .update(&canister_id, &method_name) - .with_arg(&blob) - .expire_after(timeout) - .call_and_wait(waiter_with_timeout(timeout)) - .await?; - } - } + // let walker = WalkDir::new(output_assets_path).into_iter(); + // for entry in walker { + // let entry = entry?; + // if entry.file_type().is_file() { + // let source = entry.path(); + // let relative: &Path = source + // .strip_prefix(output_assets_path) + // .expect("cannot strip prefix"); + // let content = &std::fs::read(&source)?; + // let path = relative.to_string_lossy().to_string(); + // let blob = candid::Encode!(&path, &content)?; + // + // let method_name = String::from("store"); + // + // agent + // .update(&canister_id, &method_name) + // .with_arg(&blob) + // .expire_after(timeout) + // .call_and_wait(waiter_with_timeout(timeout)) + // .await?; + // } + // } Ok(()) } From c02b6971828cf71c8deba193d9d7868b0cffadbc Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 25 Feb 2021 09:41:50 -0800 Subject: [PATCH 25/48] DFX --- src/dfx/src/lib/installers/assets.rs | 30 +++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 3f7540dd30..eea3f06bed 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -16,7 +16,7 @@ use walkdir::WalkDir; const CREATE_BATCH: &str = "create_batch"; const CREATE_CHUNK: &str = "create_chunk"; const COMMIT_BATCH: &str = "commit_batch"; -const MAX_CHUNK_SIZE: usize = 300; +const MAX_CHUNK_SIZE: usize = 1_900_000; #[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] struct CreateBatchRequest {} @@ -103,6 +103,28 @@ struct ChunkedAsset { chunk_ids: Vec, } +// async fn create_chunk(agent: &Agent, +// canister_id: &Principal, +// timeout: Duration, +// batch_id: u128, +// content: &[u8]) -> DfxResult { +// let args = CreateChunkRequest { batch_id, content }; +// let args = candid::Encode!(&args).expect("unable to encode create_chunk argument"); +// println!("create chunk"); +// agent +// .update(&canister_id, CREATE_CHUNK) +// .with_arg(args) +// .expire_after(timeout) +// .call_and_wait(waiter_with_timeout(timeout)) +// .await +// .map_err(DfxError::from) +// .and_then(|response| { +// candid::Decode!(&response, CreateChunkResponse) +// .map_err(DfxError::from) +// .map(|x| x.chunk_id) +// }) +// +// } async fn make_chunked_asset( agent: &Agent, canister_id: &Principal, @@ -154,9 +176,11 @@ async fn make_chunked_assets( ) -> DfxResult> { let futs: Vec<_> = locs .into_iter() - .map(|loc| async { make_chunked_asset(agent, canister_id, timeout, batch_id, loc).await }) + .map(|loc| make_chunked_asset(agent, canister_id, timeout, batch_id, loc)) .collect(); - try_join_all(futs).await + let fut = try_join_all(futs); + let result = fut.await; + result } async fn commit_batch( From c7c763e5dbbc61d0fae8e4b28c3bea1aa8a846f8 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 25 Feb 2021 10:16:41 -0800 Subject: [PATCH 26/48] DFX smaller async fns --- src/dfx/src/lib/installers/assets.rs | 79 ++++++++++++---------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index eea3f06bed..c04ffe5bf8 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -103,28 +103,29 @@ struct ChunkedAsset { chunk_ids: Vec, } -// async fn create_chunk(agent: &Agent, -// canister_id: &Principal, -// timeout: Duration, -// batch_id: u128, -// content: &[u8]) -> DfxResult { -// let args = CreateChunkRequest { batch_id, content }; -// let args = candid::Encode!(&args).expect("unable to encode create_chunk argument"); -// println!("create chunk"); -// agent -// .update(&canister_id, CREATE_CHUNK) -// .with_arg(args) -// .expire_after(timeout) -// .call_and_wait(waiter_with_timeout(timeout)) -// .await -// .map_err(DfxError::from) -// .and_then(|response| { -// candid::Decode!(&response, CreateChunkResponse) -// .map_err(DfxError::from) -// .map(|x| x.chunk_id) -// }) -// -// } +async fn create_chunk( + agent: &Agent, + canister_id: &Principal, + timeout: Duration, + batch_id: u128, + content: &[u8], +) -> DfxResult { + let args = CreateChunkRequest { batch_id, content }; + let args = candid::Encode!(&args)?; + println!("create chunk"); + agent + .update(&canister_id, CREATE_CHUNK) + .with_arg(args) + .expire_after(timeout) + .call_and_wait(waiter_with_timeout(timeout)) + .await + .map_err(DfxError::from) + .and_then(|response| { + candid::Decode!(&response, CreateChunkResponse) + .map_err(DfxError::from) + .map(|x| x.chunk_id) + }) +} async fn make_chunked_asset( agent: &Agent, canister_id: &Principal, @@ -139,23 +140,7 @@ async fn make_chunked_asset( ); let chunks_futures: Vec<_> = content .chunks(MAX_CHUNK_SIZE) - .map(|content| async move { - let args = CreateChunkRequest { batch_id, content }; - let args = candid::Encode!(&args).expect("unable to encode create_chunk argument"); - println!("create chunk"); - agent - .update(&canister_id, CREATE_CHUNK) - .with_arg(args) - .expire_after(timeout) - .call_and_wait(waiter_with_timeout(timeout)) - .await - .map_err(DfxError::from) - .and_then(|response| { - candid::Decode!(&response, CreateChunkResponse) - .map_err(DfxError::from) - .map(|x| x.chunk_id) - }) - }) + .map(|content| create_chunk(agent, canister_id, timeout, batch_id, content)) .collect(); println!("await chunk creation"); @@ -193,20 +178,23 @@ async fn commit_batch( let operations: Vec<_> = chunked_assets .into_iter() .map(|chunked_asset| { + let key = chunked_asset + .asset_location + .relative + .to_string_lossy() + .to_string(); vec![ - BatchOperationKind::DeleteAsset(DeleteAssetArguments { - key: chunked_asset.asset_location.relative.to_string_lossy().to_string(), - }), + BatchOperationKind::DeleteAsset(DeleteAssetArguments { key: key.clone() }), BatchOperationKind::CreateAsset(CreateAssetArguments { - key: chunked_asset.asset_location.relative.to_string_lossy().to_string(), + key: key.clone(), content_type: "application/octet-stream".to_string(), }), BatchOperationKind::SetAssetContent(SetAssetContentArguments { - key: chunked_asset.asset_location.relative.to_string_lossy().to_string(), + key, content_encoding: "identity".to_string(), chunk_ids: chunked_asset.chunk_ids, }), - ] + ] }) .flatten() .collect(); @@ -253,6 +241,7 @@ pub async fn post_install_store_assets( let canister_id = info.get_canister_id().expect("Could not find canister ID."); let batch_id = create_batch(agent, &canister_id, timeout).await?; + println!("created batch {}", batch_id); let chunked_assets = make_chunked_assets(agent, &canister_id, timeout, batch_id, asset_locations).await?; From 2adca2b5c3c76da08d92eb17be3fbb7418105146 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 25 Feb 2021 10:18:51 -0800 Subject: [PATCH 27/48] DFX clippy --- src/dfx/src/lib/installers/assets.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index c04ffe5bf8..52244a5c66 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -163,9 +163,7 @@ async fn make_chunked_assets( .into_iter() .map(|loc| make_chunked_asset(agent, canister_id, timeout, batch_id, loc)) .collect(); - let fut = try_join_all(futs); - let result = fut.await; - result + try_join_all(futs).await } async fn commit_batch( From ec8c50ae74301ba6573f7083209e0b7ec895704f Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 26 Feb 2021 16:37:22 -0800 Subject: [PATCH 28/48] DFX --- src/dfx/src/lib/installers/assets.rs | 68 +++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 52244a5c66..9725d3afa5 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -4,12 +4,14 @@ use crate::lib::error::{DfxError, DfxResult}; use crate::lib::waiter::waiter_with_timeout; use candid::{CandidType, Decode, Encode}; +use delay::{Delay, Waiter}; use futures::future::try_join_all; use ic_agent::Agent; use ic_types::Principal; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::time::Duration; +use tokio::task; use walkdir::WalkDir; //const GET: &str = "get"; @@ -113,19 +115,43 @@ async fn create_chunk( let args = CreateChunkRequest { batch_id, content }; let args = candid::Encode!(&args)?; println!("create chunk"); - agent - .update(&canister_id, CREATE_CHUNK) - .with_arg(args) - .expire_after(timeout) - .call_and_wait(waiter_with_timeout(timeout)) - .await - .map_err(DfxError::from) - .and_then(|response| { - candid::Decode!(&response, CreateChunkResponse) - .map_err(DfxError::from) - .map(|x| x.chunk_id) - }) + + let mut waiter = Delay::builder() + .timeout(std::time::Duration::from_secs(30)) + .throttle(std::time::Duration::from_secs(1)) + .build(); + waiter.start(); + + loop { + match agent + .update(&canister_id, CREATE_CHUNK) + .with_arg(&args) + .expire_after(timeout) + .call_and_wait(waiter_with_timeout(timeout)) + .await + .map_err(DfxError::from) + .and_then(|response| { + candid::Decode!(&response, CreateChunkResponse) + .map_err(DfxError::from) + .map(|x| x.chunk_id) + }) { + Ok(chunk_id) => { + println!("created chunk {}", chunk_id); + break Ok(chunk_id); + } + Err(agent_err) => { + println!("agent error ({}) waiting to retry...", agent_err); + match waiter.wait() { + Ok(()) => { + println!("retrying..."); + } + Err(_) => break Err(agent_err), + } + } + } + } } + async fn make_chunked_asset( agent: &Agent, canister_id: &Principal, @@ -138,6 +164,24 @@ async fn make_chunked_asset( "create chunks for {}", asset_location.source.to_string_lossy() ); + + // how to deal with lifetimes for agent and canister_id here + // this function won't exit until after the task is joined... + // let chunks_future_tasks: Vec<_> = content + // .chunks(MAX_CHUNK_SIZE) + // .map(|content| task::spawn(create_chunk(agent, canister_id, timeout, batch_id, content))) + // .collect(); + // println!("await chunk creation"); + // let but_lifetimes = try_join_all(chunks_future_tasks) + // .await? + // .into_iter() + // .collect::>>() + // .map(|chunk_ids| ChunkedAsset { + // asset_location, + // chunk_ids, + // }); + + let chunks_futures: Vec<_> = content .chunks(MAX_CHUNK_SIZE) .map(|content| create_chunk(agent, canister_id, timeout, batch_id, content)) From e802ca0ba409c8e5f9904194a56327a5a7245c6b Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 1 Mar 2021 15:53:36 -0800 Subject: [PATCH 29/48] DFX: slow, imperative upload that works --- src/dfx/src/lib/installers/assets.rs | 51 ++++++++++++++++++---------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 9725d3afa5..c37e549f2a 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -165,6 +165,7 @@ async fn make_chunked_asset( asset_location.source.to_string_lossy() ); + // ?? doesn't work // how to deal with lifetimes for agent and canister_id here // this function won't exit until after the task is joined... // let chunks_future_tasks: Vec<_> = content @@ -180,20 +181,31 @@ async fn make_chunked_asset( // asset_location, // chunk_ids, // }); + // ?? doesn't work + // works (sometimes) + // let chunks_futures: Vec<_> = content + // .chunks(MAX_CHUNK_SIZE) + // .map(|content| create_chunk(agent, canister_id, timeout, batch_id, content)) + // .collect(); + // println!("await chunk creation"); + // + // try_join_all(chunks_futures) + // .await + // .map(|chunk_ids| ChunkedAsset { + // asset_location, + // chunk_ids, + // }) + // works (sometimes) - let chunks_futures: Vec<_> = content - .chunks(MAX_CHUNK_SIZE) - .map(|content| create_chunk(agent, canister_id, timeout, batch_id, content)) - .collect(); - println!("await chunk creation"); - - try_join_all(chunks_futures) - .await - .map(|chunk_ids| ChunkedAsset { - asset_location, - chunk_ids, - }) + let mut chunk_ids: Vec = vec![]; + for data_chunk in content.chunks(MAX_CHUNK_SIZE) { + chunk_ids.push(create_chunk(agent, canister_id, timeout, batch_id, data_chunk).await?); + } + Ok(ChunkedAsset { + asset_location, + chunk_ids, + }) } async fn make_chunked_assets( @@ -203,11 +215,16 @@ async fn make_chunked_assets( batch_id: u128, locs: Vec, ) -> DfxResult> { - let futs: Vec<_> = locs - .into_iter() - .map(|loc| make_chunked_asset(agent, canister_id, timeout, batch_id, loc)) - .collect(); - try_join_all(futs).await + // let futs: Vec<_> = locs + // .into_iter() + // .map(|loc| make_chunked_asset(agent, canister_id, timeout, batch_id, loc)) + // .collect(); + // try_join_all(futs).await + let mut chunked_assets = vec![]; + for loc in locs { + chunked_assets.push(make_chunked_asset(agent, canister_id, timeout, batch_id, loc).await?); + } + Ok(chunked_assets) } async fn commit_batch( From 8291900626be347c72f253992373a490a98effa6 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Tue, 2 Mar 2021 10:04:24 -0800 Subject: [PATCH 30/48] CANISTER: seperate Types.mo --- src/distributed/assetstorage.mo | 139 +++++------------- .../{ => assetstorage}/StableHashMap.mo | 0 src/distributed/assetstorage/Types.mo | 72 +++++++++ 3 files changed, 110 insertions(+), 101 deletions(-) rename src/distributed/{ => assetstorage}/StableHashMap.mo (100%) create mode 100644 src/distributed/assetstorage/Types.mo diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 3ec2d96d68..cd73904edb 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -9,12 +9,14 @@ import Nat "mo:base/Nat"; import Nat8 "mo:base/Nat8"; import Nat32 "mo:base/Nat32"; import Result "mo:base/Result"; -import SHM "StableHashMap"; +import SHM "assetstorage/StableHashMap"; +import T "assetstorage/Types"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; import Word8 "mo:base/Word8"; + shared ({caller = creator}) actor class () { // old interface: @@ -22,67 +24,12 @@ shared ({caller = creator}) actor class () { public type Contents = Blob; // new hotness - public type BatchId = Nat; - public type BlobId = Text; - public type ChunkId = Nat; - public type EncodingId = Text; - public type Key = Text; - public type ContentEncoding = Text; - public type ContentType = Text; - public type Offset = Nat; - public type TotalLength = Nat; - - - public type CreateAssetArguments = { - key: Key; - content_type: Text; - }; - public type SetAssetContentArguments = { - key: Key; - content_encoding: Text; - chunk_ids: [ChunkId] - }; - public type UnsetAssetContentArguments = { - key: Key; - content_encoding: Text; - }; - public type DeleteAssetArguments = { - key: Key; - }; - public type ClearArguments = { - }; - - public type BatchOperationKind = { - #CreateAsset: CreateAssetArguments; - #SetAssetContent: SetAssetContentArguments; - #UnsetAssetContent: UnsetAssetContentArguments; - - #DeleteAsset: DeleteAssetArguments; - - #Clear: ClearArguments; - }; - - public type CommitBatchArguments = { - batch_id: BatchId; - operations: [BatchOperationKind]; - }; stable var authorized: [Principal] = [creator]; - type AssetEncoding = { - contentEncoding: Text; - content: [Blob]; - totalLength: Nat; - }; - - type Asset = { - contentType: Text; - encodings: SHM.StableHashMap; - }; - - func getAssetEncoding(asset : Asset, acceptEncodings : [Text]) : ?AssetEncoding { + func getAssetEncoding(asset : T.Asset, acceptEncodings : [Text]) : ?T.AssetEncoding { for (acceptEncoding in acceptEncodings.vals()) { switch (encodings_manipulator.get(asset.encodings, acceptEncoding)) { case null {}; @@ -93,11 +40,11 @@ shared ({caller = creator}) actor class () { }; - //stable var asset_entries : [(Key, Asset)] = []; + //stable var asset_entries : [(T.Key, T.Asset)] = []; //let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); - stable let assets : SHM.StableHashMap = SHM.StableHashMap(); - let assets_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); - let encodings_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); + stable let assets : SHM.StableHashMap = SHM.StableHashMap(); + let assets_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); + let encodings_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); system func preupgrade() { //asset_entries := Iter.toArray(assets.entries()); @@ -107,18 +54,13 @@ shared ({caller = creator}) actor class () { //asset_entries := []; }; - type Chunk = { - batch: Batch; - content: Blob; - }; - var nextChunkId = 1; - let chunks = H.HashMap(7, Int.equal, Int.hash); + let chunks = H.HashMap(7, Int.equal, Int.hash); - func createChunk(batch: Batch, content: Blob) : ChunkId { + func createChunk(batch: T.Batch, content: Blob) : T.ChunkId { let chunkId = nextChunkId; nextChunkId += 1; - let chunk : Chunk = { + let chunk : T.Chunk = { batch = batch; content = content; }; @@ -129,28 +71,23 @@ shared ({caller = creator}) actor class () { //var nextEncodingId = 1; //let encodings = H.HashMap(7, Text.equal, Text.hash); - func takeChunk(chunkId: ChunkId): Result.Result { + func takeChunk(chunkId: T.ChunkId): Result.Result { switch (chunks.remove(chunkId)) { case null #err("chunk not found"); case (?chunk) #ok(chunk.content); } }; - type Batch = { - expiry : Time; - }; - // We track when each group of blobs should expire, // so that they don't consume space after an interrupted install. let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000; var next_batch_id = 1; - type Time = Int; - let batches = H.HashMap(7, Int.equal, Int.hash); + let batches = H.HashMap(7, Int.equal, Int.hash); - func startBatch(): BatchId { + func startBatch(): T.BatchId { let batch_id = next_batch_id; next_batch_id += 1; - let batch : Batch = { + let batch : T.Batch = { expiry = Time.now() + BATCH_EXPIRY_NANOS; }; batches.put(batch_id, batch); @@ -176,7 +113,7 @@ shared ({caller = creator}) actor class () { case (?batch) createChunk(batch, contents) }; - let create_asset_args : CreateAssetArguments = { + let create_asset_args : T.CreateAssetArguments = { key = path; content_type = "application/octet-stream" }; @@ -185,7 +122,7 @@ shared ({caller = creator}) actor class () { case (#err(msg)) throw Error.reject(msg); }; - let set_asset_content_args : SetAssetContentArguments = { + let set_asset_content_args : T.SetAssetContentArguments = { key = path; content_encoding = "identity"; chunk_ids = [ chunk_id ]; @@ -211,7 +148,7 @@ shared ({caller = creator}) actor class () { }; public query func list() : async [Path] { - let iter = Iter.map<(Text, Asset), Path>(assets_manipulator.entries(assets), func (key, _) = key); + let iter = Iter.map<(Text, T.Asset), Path>(assets_manipulator.entries(assets), func (key, _) = key); Iter.toArray(iter) }; @@ -222,7 +159,7 @@ shared ({caller = creator}) actor class () { }; public query func get(arg:{ - key: Key; + key: T.Key; accept_encodings: [Text] }) : async ( { content: Blob; @@ -249,7 +186,7 @@ shared ({caller = creator}) actor class () { }; public query func get_chunk(arg:{ - key: Key; + key: T.Key; content_encoding: Text; index: Nat; }) : async ( { @@ -271,7 +208,7 @@ shared ({caller = creator}) actor class () { }; public shared ({ caller }) func create_batch(arg: {}) : async ({ - batch_id: BatchId + batch_id: T.BatchId }) { if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -284,10 +221,10 @@ shared ({caller = creator}) actor class () { }; public shared ({ caller }) func create_chunk( arg: { - batch_id: BatchId; + batch_id: T.BatchId; content: Blob; } ) : async ({ - chunk_id: ChunkId + chunk_id: T.ChunkId }) { Debug.print("create_chunk(batch " # Int.toText(arg.batch_id) # ", " # Int.toText(arg.content.size()) # " bytes)"); if (isSafe(caller) == false) @@ -303,7 +240,7 @@ shared ({caller = creator}) actor class () { } }; - public shared ({ caller }) func commit_batch(args: CommitBatchArguments) : async () { + public shared ({ caller }) func commit_batch(args: T.CommitBatchArguments) : async () { Debug.print("commit_batch (" # Int.toText(args.operations.size()) # ")"); if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -324,7 +261,7 @@ shared ({caller = creator}) actor class () { } }; - public shared ({ caller }) func create_asset(arg: CreateAssetArguments) : async () { + public shared ({ caller }) func create_asset(arg: T.CreateAssetArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -334,13 +271,13 @@ shared ({caller = creator}) actor class () { }; }; - func createAsset(arg: CreateAssetArguments) : Result.Result<(), Text> { + func createAsset(arg: T.CreateAssetArguments) : Result.Result<(), Text> { Debug.print("createAsset(" # arg.key # ")"); switch (assets_manipulator.get(assets, arg.key)) { case null { - let asset : Asset = { + let asset : T.Asset = { contentType = arg.content_type; - encodings = SHM.StableHashMap(); + encodings = SHM.StableHashMap(); }; assets_manipulator.put(assets, arg.key, asset ); }; @@ -352,7 +289,7 @@ shared ({caller = creator}) actor class () { #ok(()) }; - public shared ({ caller }) func set_asset_content(arg: SetAssetContentArguments) : async () { + public shared ({ caller }) func set_asset_content(arg: T.SetAssetContentArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -366,14 +303,14 @@ shared ({caller = creator}) actor class () { acc + blob.size() }; - func setAssetContent(arg: SetAssetContentArguments) : Result.Result<(), Text> { + func setAssetContent(arg: T.SetAssetContentArguments) : Result.Result<(), Text> { Debug.print("setAssetContent(" # arg.key # ")"); switch (assets_manipulator.get(assets, arg.key)) { case null #err("asset not found"); case (?asset) { - switch (Array.mapResult(arg.chunk_ids, takeChunk)) { + switch (Array.mapResult(arg.chunk_ids, takeChunk)) { case (#ok(chunks)) { - let encoding : AssetEncoding = { + let encoding : T.AssetEncoding = { contentEncoding = arg.content_encoding; content = chunks; totalLength = Array.foldLeft(chunks, 0, addBlobLength); @@ -388,7 +325,7 @@ shared ({caller = creator}) actor class () { } }; - public shared ({ caller }) func unset_asset_content(args: UnsetAssetContentArguments) : async () { + public shared ({ caller }) func unset_asset_content(args: T.UnsetAssetContentArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -398,11 +335,11 @@ shared ({caller = creator}) actor class () { }; }; - func unsetAssetContent(args: UnsetAssetContentArguments) : Result.Result<(), Text> { + func unsetAssetContent(args: T.UnsetAssetContentArguments) : Result.Result<(), Text> { #err("unset_asset_content: not implemented"); }; - public shared ({ caller }) func delete_asset(args: DeleteAssetArguments) : async () { + public shared ({ caller }) func delete_asset(args: T.DeleteAssetArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -412,13 +349,13 @@ shared ({caller = creator}) actor class () { }; }; - func deleteAsset(args: DeleteAssetArguments) : Result.Result<(), Text> { + func deleteAsset(args: T.DeleteAssetArguments) : Result.Result<(), Text> { Debug.print("deleteAsset(" # args.key # ")"); assets_manipulator.delete(assets, args.key); #ok(()) }; - public shared ({ caller }) func clear(args: ClearArguments) : async () { + public shared ({ caller }) func clear(args: T.ClearArguments) : async () { if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -428,7 +365,7 @@ shared ({caller = creator}) actor class () { }; }; - func clearEverything(args: ClearArguments) : Result.Result<(), Text> { + func clearEverything(args: T.ClearArguments) : Result.Result<(), Text> { #err("clear: not implemented") }; diff --git a/src/distributed/StableHashMap.mo b/src/distributed/assetstorage/StableHashMap.mo similarity index 100% rename from src/distributed/StableHashMap.mo rename to src/distributed/assetstorage/StableHashMap.mo diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo new file mode 100644 index 0000000000..b203f9fd43 --- /dev/null +++ b/src/distributed/assetstorage/Types.mo @@ -0,0 +1,72 @@ +import SHM "StableHashMap"; +import Time "mo:base/Time"; + +module Types { + public type BatchId = Nat; + public type BlobId = Text; + public type ChunkId = Nat; + public type ContentEncoding = Text; + public type ContentType = Text; + public type EncodingId = Text; + public type Key = Text; + public type Offset = Nat; + public type Time = Int; + public type TotalLength = Nat; + + public type CreateAssetArguments = { + key: Key; + content_type: Text; + }; + public type SetAssetContentArguments = { + key: Key; + content_encoding: Text; + chunk_ids: [ChunkId] + }; + public type UnsetAssetContentArguments = { + key: Key; + content_encoding: Text; + }; + public type DeleteAssetArguments = { + key: Key; + }; + public type ClearArguments = { + }; + + public type BatchOperationKind = { + #CreateAsset: CreateAssetArguments; + #SetAssetContent: SetAssetContentArguments; + #UnsetAssetContent: UnsetAssetContentArguments; + + #DeleteAsset: DeleteAssetArguments; + + #Clear: ClearArguments; + }; + + + public type CommitBatchArguments = { + batch_id: BatchId; + operations: [BatchOperationKind]; + }; + + public type AssetEncoding = { + contentEncoding: Text; + content: [Blob]; + totalLength: Nat; + }; + + public type Asset = { + contentType: Text; + encodings: SHM.StableHashMap; + }; + + public type Chunk = { + batch: Batch; + content: Blob; + }; + + public type Batch = { + expiry : Time; + }; + + +}; \ No newline at end of file From 43d3e5b49892b6b30f2fc04fe3f6a50eb085baa7 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Tue, 2 Mar 2021 14:19:38 -0800 Subject: [PATCH 31/48] Remove StableHashMap: --- src/distributed/assetstorage.mo | 86 +++++++++++++-------------- src/distributed/assetstorage/Types.mo | 16 +++-- 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index cd73904edb..2d0f37c6ea 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -9,7 +9,6 @@ import Nat "mo:base/Nat"; import Nat8 "mo:base/Nat8"; import Nat32 "mo:base/Nat32"; import Result "mo:base/Result"; -import SHM "assetstorage/StableHashMap"; import T "assetstorage/Types"; import Text "mo:base/Text"; import Time "mo:base/Time"; @@ -19,19 +18,42 @@ import Word8 "mo:base/Word8"; shared ({caller = creator}) actor class () { - // old interface: public type Path = Text; public type Contents = Blob; - // new hotness + stable var authorized: [Principal] = [creator]; + stable var stable_assets : [(T.Key, T.StableAsset)] = []; + let assets : H.HashMap = H.HashMap(7, Text.equal, Text.hash); - stable var authorized: [Principal] = [creator]; + var nextChunkId = 1; + let chunks = H.HashMap(7, Int.equal, Int.hash); + + // We track when each group of blobs should expire, + // so that they don't consume space after an interrupted install. + let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000; + var nextBatchId = 1; + let batches = H.HashMap(7, Int.equal, Int.hash); + + system func preupgrade() { + let stableIter = Iter.map(assets.entries(), func((k: T.Key, v: T.Asset)) : ((T.Key, T.StableAsset)) { + let fa : T.StableAsset = { + contentType = v.contentType; + encodings = Iter.toArray(v.encodings.entries()); + }; + (k, fa) + }); + + stable_assets := Iter.toArray(stableIter); + }; + system func postupgrade() { + stable_assets := []; + }; func getAssetEncoding(asset : T.Asset, acceptEncodings : [Text]) : ?T.AssetEncoding { for (acceptEncoding in acceptEncodings.vals()) { - switch (encodings_manipulator.get(asset.encodings, acceptEncoding)) { + switch (asset.encodings.get(acceptEncoding)) { case null {}; case (?encodings) return ?encodings; } @@ -39,24 +61,6 @@ shared ({caller = creator}) actor class () { null }; - - //stable var asset_entries : [(T.Key, T.Asset)] = []; - //let assets = H.fromIter(asset_entries.vals(), 7, Text.equal, Text.hash); - stable let assets : SHM.StableHashMap = SHM.StableHashMap(); - let assets_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); - let encodings_manipulator = SHM.StableHashMapManipulator(7, Text.equal, Text.hash); - - system func preupgrade() { - //asset_entries := Iter.toArray(assets.entries()); - }; - - system func postupgrade() { - //asset_entries := []; - }; - - var nextChunkId = 1; - let chunks = H.HashMap(7, Int.equal, Int.hash); - func createChunk(batch: T.Batch, content: Blob) : T.ChunkId { let chunkId = nextChunkId; nextChunkId += 1; @@ -68,9 +72,6 @@ shared ({caller = creator}) actor class () { chunkId }; - //var nextEncodingId = 1; - //let encodings = H.HashMap(7, Text.equal, Text.hash); - func takeChunk(chunkId: T.ChunkId): Result.Result { switch (chunks.remove(chunkId)) { case null #err("chunk not found"); @@ -78,15 +79,10 @@ shared ({caller = creator}) actor class () { } }; - // We track when each group of blobs should expire, - // so that they don't consume space after an interrupted install. - let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000; - var next_batch_id = 1; - let batches = H.HashMap(7, Int.equal, Int.hash); func startBatch(): T.BatchId { - let batch_id = next_batch_id; - next_batch_id += 1; + let batch_id = nextBatchId; + nextBatchId += 1; let batch : T.Batch = { expiry = Time.now() + BATCH_EXPIRY_NANOS; }; @@ -134,7 +130,7 @@ shared ({caller = creator}) actor class () { }; public query func retrieve(path : Path) : async Blob { - switch (assets_manipulator.get(assets, path)) { + switch (assets.get(path)) { case null throw Error.reject("not found"); case (?asset) { switch (getAssetEncoding(asset, ["identity"])) { @@ -148,7 +144,7 @@ shared ({caller = creator}) actor class () { }; public query func list() : async [Path] { - let iter = Iter.map<(Text, T.Asset), Path>(assets_manipulator.entries(assets), func (key, _) = key); + let iter = Iter.map<(Text, T.Asset), Path>(assets.entries(), func (key, _) = key); Iter.toArray(iter) }; @@ -167,7 +163,7 @@ shared ({caller = creator}) actor class () { content_encoding: Text; total_length: Nat; } ) { - switch (assets_manipulator.get(assets, arg.key)) { + switch (assets.get(arg.key)) { case null throw Error.reject("asset not found"); case (?asset) { switch (getAssetEncoding(asset, arg.accept_encodings)) { @@ -192,10 +188,10 @@ shared ({caller = creator}) actor class () { }) : async ( { content: Blob }) { - switch (assets_manipulator.get(assets, arg.key)) { + switch (assets.get(arg.key)) { case null throw Error.reject("asset not found"); case (?asset) { - switch (encodings_manipulator.get(asset.encodings, arg.content_encoding)) { + switch (asset.encodings.get(arg.content_encoding)) { case null throw Error.reject("no such encoding"); case (?encoding) { { @@ -273,13 +269,13 @@ shared ({caller = creator}) actor class () { func createAsset(arg: T.CreateAssetArguments) : Result.Result<(), Text> { Debug.print("createAsset(" # arg.key # ")"); - switch (assets_manipulator.get(assets, arg.key)) { + switch (assets.get(arg.key)) { case null { let asset : T.Asset = { contentType = arg.content_type; - encodings = SHM.StableHashMap(); + encodings = H.HashMap(7, Text.equal, Text.hash); }; - assets_manipulator.put(assets, arg.key, asset ); + assets.put(arg.key, asset ); }; case (?asset) { if (asset.contentType != arg.content_type) @@ -305,7 +301,7 @@ shared ({caller = creator}) actor class () { func setAssetContent(arg: T.SetAssetContentArguments) : Result.Result<(), Text> { Debug.print("setAssetContent(" # arg.key # ")"); - switch (assets_manipulator.get(assets, arg.key)) { + switch (assets.get(arg.key)) { case null #err("asset not found"); case (?asset) { switch (Array.mapResult(arg.chunk_ids, takeChunk)) { @@ -316,7 +312,7 @@ shared ({caller = creator}) actor class () { totalLength = Array.foldLeft(chunks, 0, addBlobLength); }; - encodings_manipulator.put(asset.encodings, arg.content_encoding, encoding); + asset.encodings.put(arg.content_encoding, encoding); #ok(()); }; case (#err(err)) #err(err); @@ -351,7 +347,9 @@ shared ({caller = creator}) actor class () { func deleteAsset(args: T.DeleteAssetArguments) : Result.Result<(), Text> { Debug.print("deleteAsset(" # args.key # ")"); - assets_manipulator.delete(assets, args.key); + if (assets.size() > 0) { // avoid div/0 bug https://github.com/dfinity/motoko-base/issues/228 + assets.delete(args.key); + }; #ok(()) }; diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo index b203f9fd43..7c734c90f5 100644 --- a/src/distributed/assetstorage/Types.mo +++ b/src/distributed/assetstorage/Types.mo @@ -1,4 +1,4 @@ -import SHM "StableHashMap"; +import H "mo:base/HashMap"; import Time "mo:base/Time"; module Types { @@ -17,18 +17,22 @@ module Types { key: Key; content_type: Text; }; + public type SetAssetContentArguments = { key: Key; content_encoding: Text; chunk_ids: [ChunkId] }; + public type UnsetAssetContentArguments = { key: Key; content_encoding: Text; }; + public type DeleteAssetArguments = { key: Key; }; + public type ClearArguments = { }; @@ -42,7 +46,6 @@ module Types { #Clear: ClearArguments; }; - public type CommitBatchArguments = { batch_id: BatchId; operations: [BatchOperationKind]; @@ -56,7 +59,12 @@ module Types { public type Asset = { contentType: Text; - encodings: SHM.StableHashMap; + encodings: H.HashMap; + }; + + public type StableAsset = { + contentType: Text; + encodings: [(Text, AssetEncoding)]; }; public type Chunk = { @@ -67,6 +75,4 @@ module Types { public type Batch = { expiry : Time; }; - - }; \ No newline at end of file From 7de6afaa69e0b37cdac33ced7161b5d3eee9ec1a Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Tue, 2 Mar 2021 18:23:35 -0800 Subject: [PATCH 32/48] expire batches freshen batches on create_chunk implement clear verify chunk lengths are homogeneous --- src/distributed/assetstorage.mo | 111 ++++++++++++++++++++------ src/distributed/assetstorage/Types.mo | 20 ++++- src/distributed/assetstorage/Utils.mo | 12 +++ 3 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 src/distributed/assetstorage/Utils.mo diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 2d0f37c6ea..5e35c8e58e 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -13,6 +13,7 @@ import T "assetstorage/Types"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; +import U "assetstorage/Utils"; import Word8 "mo:base/Word8"; @@ -23,32 +24,24 @@ shared ({caller = creator}) actor class () { stable var authorized: [Principal] = [creator]; - stable var stable_assets : [(T.Key, T.StableAsset)] = []; - let assets : H.HashMap = H.HashMap(7, Text.equal, Text.hash); + stable var stableAssets : [(T.Key, T.StableAsset)] = []; + let assets = H.fromIter(Iter.map(stableAssets.vals(), T.fromStableAssetEntry), 7, Text.equal, Text.hash); var nextChunkId = 1; let chunks = H.HashMap(7, Int.equal, Int.hash); // We track when each group of blobs should expire, // so that they don't consume space after an interrupted install. - let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000; + let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000 * 1000; var nextBatchId = 1; let batches = H.HashMap(7, Int.equal, Int.hash); system func preupgrade() { - let stableIter = Iter.map(assets.entries(), func((k: T.Key, v: T.Asset)) : ((T.Key, T.StableAsset)) { - let fa : T.StableAsset = { - contentType = v.contentType; - encodings = Iter.toArray(v.encodings.entries()); - }; - (k, fa) - }); - - stable_assets := Iter.toArray(stableIter); + stableAssets := Iter.toArray(Iter.map(assets.entries(), T.fromAssetEntry)); }; system func postupgrade() { - stable_assets := []; + stableAssets := []; }; func getAssetEncoding(asset : T.Asset, acceptEncodings : [Text]) : ?T.AssetEncoding { @@ -68,7 +61,10 @@ shared ({caller = creator}) actor class () { batch = batch; content = content; }; + + batch.expiry := Time.now() + BATCH_EXPIRY_NANOS; chunks.put(chunkId, chunk); + chunkId }; @@ -84,12 +80,41 @@ shared ({caller = creator}) actor class () { let batch_id = nextBatchId; nextBatchId += 1; let batch : T.Batch = { - expiry = Time.now() + BATCH_EXPIRY_NANOS; + var expiry = Time.now() + BATCH_EXPIRY_NANOS; }; batches.put(batch_id, batch); batch_id }; + func expireBatches() : () { + let now = Time.now(); + let batchesToDelete: H.HashMap = H.mapFilter(batches, Int.equal, Int.hash, + func(k: Int, batch: T.Batch) : ?T.Batch { + if (batch.expiry <= now) + ?batch + else + null + } + ); + for ((k,_) in batchesToDelete.entries()) { + Debug.print("delete expired batch " # Int.toText(k)); + + batches.delete(k); + }; + let chunksToDelete = H.mapFilter(chunks, Int.equal, Int.hash, + func(k: Int, chunk: T.Chunk) : ?T.Chunk { + if (chunk.batch.expiry <= now) + ?chunk + else + null + } + ); + for ((k,_) in chunksToDelete.entries()) { + Debug.print("delete expired chunk " # Int.toText(k)); + chunks.delete(k); + }; + }; + public shared ({ caller }) func authorize(other: Principal) : async () { if (isSafe(caller)) { authorized := Array.append(authorized, [other]); @@ -149,7 +174,7 @@ shared ({caller = creator}) actor class () { }; func isSafe(caller: Principal) : Bool { - //return true; + return true; func eq(value: Principal): Bool = value == caller; Array.find(authorized, eq) != null }; @@ -210,7 +235,7 @@ shared ({caller = creator}) actor class () { throw Error.reject("not authorized"); Debug.print("create_batch"); - + expireBatches(); { batch_id = startBatch(); } @@ -299,6 +324,29 @@ shared ({caller = creator}) actor class () { acc + blob.size() }; + func chunkLengthsMatch(chunks: [Blob]): Bool { + if (chunks.size() == 0) + return true; + + let expectedLength = chunks[0].size(); + for (i in Iter.range(1, chunks.size()-2)) { + Debug.print("chunk at index " # Int.toText(i) # " has length " # Int.toText(chunks[i].size()) # " and expected is " # Int.toText(expectedLength) ); + if (chunks[i].size() != expectedLength) { + Debug.print("chunk at index " # Int.toText(i) # " with length " # Int.toText(chunks[i].size()) # " does not match expected length " # Int.toText(expectedLength) ); + + return false; + } + }; + //var i = 1; + //var last = chunks.size() - 1; + //while (i <= last) { + // if (chunks[i].size() != expectedLength) + // return false; + // i += 1; + //}; + true + }; + func setAssetContent(arg: T.SetAssetContentArguments) : Result.Result<(), Text> { Debug.print("setAssetContent(" # arg.key # ")"); switch (assets.get(arg.key)) { @@ -306,14 +354,18 @@ shared ({caller = creator}) actor class () { case (?asset) { switch (Array.mapResult(arg.chunk_ids, takeChunk)) { case (#ok(chunks)) { - let encoding : T.AssetEncoding = { - contentEncoding = arg.content_encoding; - content = chunks; - totalLength = Array.foldLeft(chunks, 0, addBlobLength); + if (chunkLengthsMatch(chunks) == false) { + #err("chunk lengths do not match the size of the first chunk") + } else { + let encoding : T.AssetEncoding = { + contentEncoding = arg.content_encoding; + content = chunks; + totalLength = Array.foldLeft(chunks, 0, addBlobLength); + }; + + asset.encodings.put(arg.content_encoding, encoding); + #ok(()); }; - - asset.encodings.put(arg.content_encoding, encoding); - #ok(()); }; case (#err(err)) #err(err); }; @@ -364,9 +416,18 @@ shared ({caller = creator}) actor class () { }; func clearEverything(args: T.ClearArguments) : Result.Result<(), Text> { - #err("clear: not implemented") + stableAssets := []; + U.clearHashMap(assets); + + nextBatchId := 1; + U.clearHashMap(batches); + + nextChunkId := 1; + U.clearHashMap(chunks); + + #ok(()) }; - public func version_13() : async() { + public func version_14() : async() { } }; diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo index 7c734c90f5..3c5b9c92f2 100644 --- a/src/distributed/assetstorage/Types.mo +++ b/src/distributed/assetstorage/Types.mo @@ -1,4 +1,6 @@ import H "mo:base/HashMap"; +import Iter "mo:base/Iter"; +import Text "mo:base/Text"; import Time "mo:base/Time"; module Types { @@ -67,12 +69,28 @@ module Types { encodings: [(Text, AssetEncoding)]; }; + public func fromAssetEntry((k: Key, v: Asset)) : ((Key, StableAsset)) { + let fa : StableAsset = { + contentType = v.contentType; + encodings = Iter.toArray(v.encodings.entries()); + }; + (k, fa) + }; + + public func fromStableAssetEntry((k: Key, v: StableAsset)) : ((Key, Asset)) { + let a : Asset = { + contentType = v.contentType; + encodings = H.fromIter(v.encodings.vals(), 7, Text.equal, Text.hash); + }; + (k, a) + }; + public type Chunk = { batch: Batch; content: Blob; }; public type Batch = { - expiry : Time; + var expiry : Time; }; }; \ No newline at end of file diff --git a/src/distributed/assetstorage/Utils.mo b/src/distributed/assetstorage/Utils.mo new file mode 100644 index 0000000000..5fd6b74c27 --- /dev/null +++ b/src/distributed/assetstorage/Utils.mo @@ -0,0 +1,12 @@ +import H "mo:base/HashMap"; +import Iter "mo:base/Iter"; + +module Utils { + + public func clearHashMap(h:H.HashMap) : () { + let keys = Iter.toArray(Iter.map(h.entries(), func((k: K, v: V)): K { k })); + for (key in keys.vals()) { + h.delete(key); + }; + }; +}; \ No newline at end of file From 28c39489b73d437e828fed1394948538c35b5cdd Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 4 Mar 2021 13:58:32 -0800 Subject: [PATCH 33/48] e2e test - arbitrarily large files --- e2e/tests-dfx/assetscanister.bash | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index c478702487..05f5de6298 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -14,6 +14,7 @@ teardown() { } @test "can store and retrieve assets by key" { + skip install_asset assetscanister dfx_start @@ -46,7 +47,7 @@ teardown() { HOME=. assert_command_fail dfx canister call --update e2e_project_assets store '("index.js", vec { 1; 2; 3; })' } -@test 'new asset canister interface' { +@test 'can store arbitrarily large files' { install_asset assetscanister dfx_start @@ -54,27 +55,18 @@ teardown() { dfx build dfx canister install e2e_project_assets - assert_command dfx canister call --query e2e_project_assets retrieve '("binary/noise.txt")' --output idl - assert_eq '(blob "\b8\01 \80\0aw12 \00xy\0aKL\0b\0ajk")' - - assert_command dfx canister call --query e2e_project_assets retrieve '("text-with-newlines.txt")' --output idl - assert_eq '(blob "cherries\0ait'\''s cherry season\0aCHERRIES")' - - assert_command dfx canister call --update e2e_project_assets store '("AA", blob "hello, world!")' - assert_eq '()' - assert_command dfx canister call --update e2e_project_assets store '("B", vec { 88; 87; 86; })' - assert_eq '()' + dd if=/dev/urandom of=src/e2e_project_assets/assets/large-asset.bin bs=1000000 count=25 - assert_command dfx canister call --query e2e_project_assets retrieve '("B")' --output idl - assert_eq '(blob "XWV")' + dfx deploy - assert_command dfx canister call --query e2e_project_assets retrieve '("AA")' --output idl - assert_eq '(blob "hello, world!")' + assert_command dfx canister call --query e2e_project_assets get '(record{key="large-asset.bin";accept_encodings=vec{"identity"}})' + assert_match 'total_length = 25_000_000' + assert_match 'content_type = "application/octet-stream"' + assert_match 'content_encoding = "identity"' - assert_command dfx canister call --query e2e_project_assets retrieve '("B")' --output idl - assert_eq '(blob "XWV")' + assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="large-asset.bin";content_encoding="identity";index=4})' - assert_command_fail dfx canister call --query e2e_project_assets retrieve '("C")' + assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="large-asset.bin";content_encoding="identity";index=13})' + assert_command_fail dfx canister call --query e2e_project_assets get_chunk '(record{key="large-asset.bin";content_encoding="identity";index=14})' - HOME=. assert_command_fail dfx canister call --update e2e_project_assets store '("index.js", vec { 1; 2; 3; })' } \ No newline at end of file From 2f6ea3feaa61f3c1020649f5532737e9b537740d Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 4 Mar 2021 13:59:01 -0800 Subject: [PATCH 34/48] dfx: comment out imports --- src/dfx/src/lib/installers/assets.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index c37e549f2a..fe62d03798 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -5,13 +5,13 @@ use crate::lib::waiter::waiter_with_timeout; use candid::{CandidType, Decode, Encode}; use delay::{Delay, Waiter}; -use futures::future::try_join_all; +//use futures::future::try_join_all; use ic_agent::Agent; use ic_types::Principal; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::time::Duration; -use tokio::task; +//use tokio::task; use walkdir::WalkDir; //const GET: &str = "get"; From ecba19f2ea28c0ced7954e3edc1f5ae5ee7e2af7 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 4 Mar 2021 13:59:22 -0800 Subject: [PATCH 35/48] canister: Batch class --- src/distributed/assetstorage.mo | 23 ++++++++++------------- src/distributed/assetstorage/Types.mo | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage.mo index 5e35c8e58e..f4f2dc8fd6 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage.mo @@ -1,5 +1,4 @@ import Array "mo:base/Array"; -import Buffer "mo:base/Buffer"; import Debug "mo:base/Debug"; import Error "mo:base/Error"; import H "mo:base/HashMap"; @@ -7,14 +6,12 @@ import Int "mo:base/Int"; import Iter "mo:base/Iter"; import Nat "mo:base/Nat"; import Nat8 "mo:base/Nat8"; -import Nat32 "mo:base/Nat32"; import Result "mo:base/Result"; import T "assetstorage/Types"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; import U "assetstorage/Utils"; -import Word8 "mo:base/Word8"; shared ({caller = creator}) actor class () { @@ -32,7 +29,7 @@ shared ({caller = creator}) actor class () { // We track when each group of blobs should expire, // so that they don't consume space after an interrupted install. - let BATCH_EXPIRY_NANOS = 5 * 60 * 1000 * 1000 * 1000; + //let batchExpiryNanos = 5 * 60 * 1000 * 1000 * 1000; var nextBatchId = 1; let batches = H.HashMap(7, Int.equal, Int.hash); @@ -48,7 +45,7 @@ shared ({caller = creator}) actor class () { for (acceptEncoding in acceptEncodings.vals()) { switch (asset.encodings.get(acceptEncoding)) { case null {}; - case (?encodings) return ?encodings; + case (?encoding) return ?encoding; } }; null @@ -62,7 +59,7 @@ shared ({caller = creator}) actor class () { content = content; }; - batch.expiry := Time.now() + BATCH_EXPIRY_NANOS; + batch.refreshExpiry(); chunks.put(chunkId, chunk); chunkId @@ -79,9 +76,7 @@ shared ({caller = creator}) actor class () { func startBatch(): T.BatchId { let batch_id = nextBatchId; nextBatchId += 1; - let batch : T.Batch = { - var expiry = Time.now() + BATCH_EXPIRY_NANOS; - }; + let batch = T.Batch(); batches.put(batch_id, batch); batch_id }; @@ -90,7 +85,7 @@ shared ({caller = creator}) actor class () { let now = Time.now(); let batchesToDelete: H.HashMap = H.mapFilter(batches, Int.equal, Int.hash, func(k: Int, batch: T.Batch) : ?T.Batch { - if (batch.expiry <= now) + if (batch.expired(now)) ?batch else null @@ -103,7 +98,7 @@ shared ({caller = creator}) actor class () { }; let chunksToDelete = H.mapFilter(chunks, Int.equal, Int.hash, func(k: Int, chunk: T.Chunk) : ?T.Chunk { - if (chunk.batch.expiry <= now) + if (chunk.batch.expired(now)) ?chunk else null @@ -128,7 +123,7 @@ shared ({caller = creator}) actor class () { throw Error.reject("not authorized"); }; - let batch_id = startBatch(); + let batch_id = startBatch(); // ew let chunk_id = switch (batches.get(batch_id)) { case null throw Error.reject("batch not found"); case (?batch) createChunk(batch, contents) @@ -236,6 +231,7 @@ shared ({caller = creator}) actor class () { Debug.print("create_batch"); expireBatches(); + { batch_id = startBatch(); } @@ -279,7 +275,8 @@ shared ({caller = creator}) actor class () { case (#ok(())) {}; case (#err(msg)) throw Error.reject(msg); }; - } + }; + batches.delete(args.batch_id); }; public shared ({ caller }) func create_asset(arg: T.CreateAssetArguments) : async () { diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo index 3c5b9c92f2..312b0d0635 100644 --- a/src/distributed/assetstorage/Types.mo +++ b/src/distributed/assetstorage/Types.mo @@ -90,7 +90,23 @@ module Types { content: Blob; }; - public type Batch = { - var expiry : Time; + object batch { + let expiryNanos = 300_000_000_000; // 5 * 60 * 1000 * 1000 * 1000; + + public func nextExpireTime() : Time { + Time.now() + expiryNanos + } + }; + + public class Batch() { + var expiresAt : Time = batch.nextExpireTime(); + + public func refreshExpiry() { + expiresAt := batch.nextExpireTime(); + }; + + public func expired(asOf : Time) : Bool { + expiresAt <= asOf + }; }; }; \ No newline at end of file From 4b04f3f07325a3418e2619d7ffb3f631f4a8d2f9 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 5 Mar 2021 09:39:32 -0800 Subject: [PATCH 36/48] Move assetstorage.mo to assetstore/Main.mo Add Chunks.mo and Batches.mo --- distributed-canisters.nix | 8 +- src/distributed/assetstorage/Batches.mo | 60 +++++++++ src/distributed/assetstorage/Chunks.mo | 67 ++++++++++ .../{assetstorage.mo => assetstorage/Main.mo} | 115 ++++-------------- src/distributed/assetstorage/Types.mo | 3 + src/distributed/assetstorage/Utils.mo | 27 +++- 6 files changed, 184 insertions(+), 96 deletions(-) create mode 100644 src/distributed/assetstorage/Batches.mo create mode 100644 src/distributed/assetstorage/Chunks.mo rename src/distributed/{assetstorage.mo => assetstorage/Main.mo} (77%) diff --git a/distributed-canisters.nix b/distributed-canisters.nix index b54b966b17..52b9e64866 100644 --- a/distributed-canisters.nix +++ b/distributed-canisters.nix @@ -12,19 +12,19 @@ pkgs.runCommandNoCCLocal "distributed-canisters" { } '' mkdir -p $out - for canister_mo in ${distributed}/*.mo; do - canister_name=$(basename -s .mo $canister_mo) + for canister_dir in $(find ${distributed} -mindepth 1 -maxdepth 1 -type d); do + canister_name=$(basename $canister_dir) build_dir=$out/$canister_name mkdir -p $build_dir $moc/bin/moc \ - $canister_mo \ + $canister_dir/Main.mo \ -o $build_dir/$canister_name.did \ --idl \ --package base $base $moc/bin/moc \ - $canister_mo \ + $canister_dir/Main.mo \ -o $build_dir/$canister_name.wasm \ -c --release \ --package base $base diff --git a/src/distributed/assetstorage/Batches.mo b/src/distributed/assetstorage/Batches.mo new file mode 100644 index 0000000000..b652db5b24 --- /dev/null +++ b/src/distributed/assetstorage/Batches.mo @@ -0,0 +1,60 @@ +import Debug "mo:base/Debug"; +import Int "mo:base/Int"; +import Time "mo:base/Time"; + +import H "mo:base/HashMap"; +import T "Types"; +import U "Utils"; + +module { + +public class Batches() { + // We group the staged chunks into batches. Uploading a chunk refreshes the batch's expiry timer. + // We delete expired batches so that they don't consume space after an interrupted install. + var nextBatchId = 1; + let batches = H.HashMap(7, Int.equal, Int.hash); + + public func get(batchId: T.BatchId) : ?T.Batch { + batches.get(batchId) + }; + public func delete(batchId: T.BatchId) { + batches.delete(batchId) + }; + + public func startBatch(): T.BatchId { + let batch_id = nextBatchId; + nextBatchId += 1; + let batch = T.Batch(); + batches.put(batch_id, batch); + batch_id + }; + + public func deleteExpired() : () { + let now = Time.now(); + /*let batchesToDelete: H.HashMap = H.mapFilter(batches, Int.equal, Int.hash, + func(k: Int, batch: T.Batch) : ?T.Batch { + if (batch.expired(now)) + ?batch + else + null + } + ); + for ((k,_) in batchesToDelete.entries()) { + Debug.print("delete expired batch " # Int.toText(k)); + + batches.delete(k); + };*/ + U.deleteFromHashMap(batches, Int.equal, Int.hash, + func(k: Int, batch: T.Batch) : Bool { + batch.expired(now) + } + ); + }; + + public func reset() { + nextBatchId := 1; + U.clearHashMap(batches); + } +} + +} \ No newline at end of file diff --git a/src/distributed/assetstorage/Chunks.mo b/src/distributed/assetstorage/Chunks.mo new file mode 100644 index 0000000000..4db63934db --- /dev/null +++ b/src/distributed/assetstorage/Chunks.mo @@ -0,0 +1,67 @@ +import Debug "mo:base/Debug"; +import HashMap "mo:base/HashMap"; +import Int "mo:base/Int"; +import Result "mo:base/Result"; +import Time "mo:base/Time"; + +import T "Types"; +import U "Utils"; + +module { + +public class Chunks() { + // We stage asset content chunks here, + // before assigning them to asset content encodings. + var nextChunkId = 1; + let chunks = HashMap.HashMap(7, Int.equal, Int.hash); + + public func create(batch: T.Batch, content: Blob) : T.ChunkId { + let chunkId = nextChunkId; + nextChunkId += 1; + let chunk : T.Chunk = { + batch = batch; + content = content; + }; + + batch.refreshExpiry(); + chunks.put(chunkId, chunk); + + chunkId + }; + + public func take(chunkId: T.ChunkId): Result.Result { + switch (chunks.remove(chunkId)) { + case null #err("chunk not found"); + case (?chunk) #ok(chunk.content); + } + }; + + public func reset() { + nextChunkId := 1; + U.clearHashMap(chunks); + }; + + public func deleteExpired() : () { + let now = Time.now(); + /*let chunksToDelete = HashMap.mapFilter(chunks, Int.equal, Int.hash, + func(k: Int, chunk: T.Chunk) : ?T.Chunk { + if (chunk.batch.expired(now)) + ?chunk + else + null + } + ); + for ((k,_) in chunksToDelete.entries()) { + Debug.print("delete expired chunk " # Int.toText(k)); + chunks.delete(k); + };*/ + U.deleteFromHashMap(chunks, Int.equal, Int.hash, + func(k: Int, chunk: T.Chunk) : Bool { + chunk.batch.expired(now) + } + ); + + }; +} + +} \ No newline at end of file diff --git a/src/distributed/assetstorage.mo b/src/distributed/assetstorage/Main.mo similarity index 77% rename from src/distributed/assetstorage.mo rename to src/distributed/assetstorage/Main.mo index f4f2dc8fd6..cc6c70cce0 100644 --- a/src/distributed/assetstorage.mo +++ b/src/distributed/assetstorage/Main.mo @@ -1,37 +1,30 @@ import Array "mo:base/Array"; import Debug "mo:base/Debug"; import Error "mo:base/Error"; -import H "mo:base/HashMap"; +import HashMap "mo:base/HashMap"; import Int "mo:base/Int"; import Iter "mo:base/Iter"; import Nat "mo:base/Nat"; import Nat8 "mo:base/Nat8"; import Result "mo:base/Result"; -import T "assetstorage/Types"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; -import U "assetstorage/Utils"; +import B "Batches"; +import C "Chunks"; +import T "Types"; +import U "Utils"; shared ({caller = creator}) actor class () { - public type Path = Text; - public type Contents = Blob; - stable var authorized: [Principal] = [creator]; stable var stableAssets : [(T.Key, T.StableAsset)] = []; - let assets = H.fromIter(Iter.map(stableAssets.vals(), T.fromStableAssetEntry), 7, Text.equal, Text.hash); - - var nextChunkId = 1; - let chunks = H.HashMap(7, Int.equal, Int.hash); + let assets = HashMap.fromIter(Iter.map(stableAssets.vals(), T.fromStableAssetEntry), 7, Text.equal, Text.hash); - // We track when each group of blobs should expire, - // so that they don't consume space after an interrupted install. - //let batchExpiryNanos = 5 * 60 * 1000 * 1000 * 1000; - var nextBatchId = 1; - let batches = H.HashMap(7, Int.equal, Int.hash); + let chunks = C.Chunks(); + let batches = B.Batches(); system func preupgrade() { stableAssets := Iter.toArray(Iter.map(assets.entries(), T.fromAssetEntry)); @@ -51,64 +44,6 @@ shared ({caller = creator}) actor class () { null }; - func createChunk(batch: T.Batch, content: Blob) : T.ChunkId { - let chunkId = nextChunkId; - nextChunkId += 1; - let chunk : T.Chunk = { - batch = batch; - content = content; - }; - - batch.refreshExpiry(); - chunks.put(chunkId, chunk); - - chunkId - }; - - func takeChunk(chunkId: T.ChunkId): Result.Result { - switch (chunks.remove(chunkId)) { - case null #err("chunk not found"); - case (?chunk) #ok(chunk.content); - } - }; - - - func startBatch(): T.BatchId { - let batch_id = nextBatchId; - nextBatchId += 1; - let batch = T.Batch(); - batches.put(batch_id, batch); - batch_id - }; - - func expireBatches() : () { - let now = Time.now(); - let batchesToDelete: H.HashMap = H.mapFilter(batches, Int.equal, Int.hash, - func(k: Int, batch: T.Batch) : ?T.Batch { - if (batch.expired(now)) - ?batch - else - null - } - ); - for ((k,_) in batchesToDelete.entries()) { - Debug.print("delete expired batch " # Int.toText(k)); - - batches.delete(k); - }; - let chunksToDelete = H.mapFilter(chunks, Int.equal, Int.hash, - func(k: Int, chunk: T.Chunk) : ?T.Chunk { - if (chunk.batch.expired(now)) - ?chunk - else - null - } - ); - for ((k,_) in chunksToDelete.entries()) { - Debug.print("delete expired chunk " # Int.toText(k)); - chunks.delete(k); - }; - }; public shared ({ caller }) func authorize(other: Principal) : async () { if (isSafe(caller)) { @@ -118,15 +53,15 @@ shared ({caller = creator}) actor class () { } }; - public shared ({ caller }) func store(path : Path, contents : Contents) : async () { + public shared ({ caller }) func store(path : T.Path, contents : T.Contents) : async () { if (isSafe(caller) == false) { throw Error.reject("not authorized"); }; - let batch_id = startBatch(); // ew + let batch_id = batches.startBatch(); // ew let chunk_id = switch (batches.get(batch_id)) { case null throw Error.reject("batch not found"); - case (?batch) createChunk(batch, contents) + case (?batch) chunks.create(batch, contents) }; let create_asset_args : T.CreateAssetArguments = { @@ -149,7 +84,7 @@ shared ({caller = creator}) actor class () { }; }; - public query func retrieve(path : Path) : async Blob { + public query func retrieve(path : T.Path) : async T.Contents { switch (assets.get(path)) { case null throw Error.reject("not found"); case (?asset) { @@ -163,8 +98,8 @@ shared ({caller = creator}) actor class () { } }; - public query func list() : async [Path] { - let iter = Iter.map<(Text, T.Asset), Path>(assets.entries(), func (key, _) = key); + public query func list() : async [T.Path] { + let iter = Iter.map<(Text, T.Asset), T.Path>(assets.entries(), func (key, _) = key); Iter.toArray(iter) }; @@ -230,10 +165,12 @@ shared ({caller = creator}) actor class () { throw Error.reject("not authorized"); Debug.print("create_batch"); - expireBatches(); + + batches.deleteExpired(); + chunks.deleteExpired(); { - batch_id = startBatch(); + batch_id = batches.startBatch(); } }; @@ -251,7 +188,7 @@ shared ({caller = creator}) actor class () { case null throw Error.reject("batch not found"); case (?batch) { { - chunk_id = createChunk(batch, arg.content) + chunk_id = chunks.create(batch, arg.content) } } } @@ -295,7 +232,7 @@ shared ({caller = creator}) actor class () { case null { let asset : T.Asset = { contentType = arg.content_type; - encodings = H.HashMap(7, Text.equal, Text.hash); + encodings = HashMap.HashMap(7, Text.equal, Text.hash); }; assets.put(arg.key, asset ); }; @@ -349,7 +286,7 @@ shared ({caller = creator}) actor class () { switch (assets.get(arg.key)) { case null #err("asset not found"); case (?asset) { - switch (Array.mapResult(arg.chunk_ids, takeChunk)) { + switch (Array.mapResult(arg.chunk_ids, chunks.take)) { case (#ok(chunks)) { if (chunkLengthsMatch(chunks) == false) { #err("chunk lengths do not match the size of the first chunk") @@ -416,15 +353,13 @@ shared ({caller = creator}) actor class () { stableAssets := []; U.clearHashMap(assets); - nextBatchId := 1; - U.clearHashMap(batches); - - nextChunkId := 1; - U.clearHashMap(chunks); + batches.reset(); + chunks.reset(); #ok(()) }; public func version_14() : async() { - } + }; + }; diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo index 312b0d0635..2993f83188 100644 --- a/src/distributed/assetstorage/Types.mo +++ b/src/distributed/assetstorage/Types.mo @@ -4,6 +4,9 @@ import Text "mo:base/Text"; import Time "mo:base/Time"; module Types { + public type Contents = Blob; + public type Path = Text; + public type BatchId = Nat; public type BlobId = Text; public type ChunkId = Nat; diff --git a/src/distributed/assetstorage/Utils.mo b/src/distributed/assetstorage/Utils.mo index 5fd6b74c27..ec7238d112 100644 --- a/src/distributed/assetstorage/Utils.mo +++ b/src/distributed/assetstorage/Utils.mo @@ -1,12 +1,35 @@ -import H "mo:base/HashMap"; +import Debug "mo:base/Debug"; +import Hash "mo:base/Hash"; +import HashMap "mo:base/HashMap"; +import Int "mo:base/Int"; import Iter "mo:base/Iter"; module Utils { - public func clearHashMap(h:H.HashMap) : () { + public func clearHashMap(h:HashMap.HashMap) : () { let keys = Iter.toArray(Iter.map(h.entries(), func((k: K, v: V)): K { k })); for (key in keys.vals()) { h.delete(key); }; }; + + public func deleteFromHashMap + (h:HashMap.HashMap, + keyEq: (K,K) -> Bool, + keyHash: K -> Hash.Hash, + deleteFn: (K, V) -> Bool): () { + let entriesToDelete: HashMap.HashMap = HashMap.mapFilter(h, keyEq, keyHash, + func(k: K, v: V) : ?V { + if (deleteFn(k, v)) + ?v + else + null + } + ); + for ((k,_) in entriesToDelete.entries()) { + Debug.print("delete expired"); + h.delete(k); + }; + + } }; \ No newline at end of file From 408330c9d6614cdb639071573660a7e7563e9e25 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 5 Mar 2021 14:30:17 -0800 Subject: [PATCH 37/48] Asset.mo --- src/distributed/assetstorage/Asset.mo | 68 +++++++++ src/distributed/assetstorage/Batch.mo | 69 +++++++++ src/distributed/assetstorage/Batches.mo | 60 -------- src/distributed/assetstorage/Chunk.mo | 63 ++++++++ src/distributed/assetstorage/Chunks.mo | 67 --------- src/distributed/assetstorage/Main.mo | 62 ++++---- src/distributed/assetstorage/StableHashMap.mo | 142 ------------------ src/distributed/assetstorage/Types.mo | 68 --------- 8 files changed, 226 insertions(+), 373 deletions(-) create mode 100644 src/distributed/assetstorage/Asset.mo create mode 100644 src/distributed/assetstorage/Batch.mo delete mode 100644 src/distributed/assetstorage/Batches.mo create mode 100644 src/distributed/assetstorage/Chunk.mo delete mode 100644 src/distributed/assetstorage/Chunks.mo delete mode 100644 src/distributed/assetstorage/StableHashMap.mo diff --git a/src/distributed/assetstorage/Asset.mo b/src/distributed/assetstorage/Asset.mo new file mode 100644 index 0000000000..cc0dbe7797 --- /dev/null +++ b/src/distributed/assetstorage/Asset.mo @@ -0,0 +1,68 @@ +import HashMap "mo:base/HashMap"; +import Iter "mo:base/Iter"; +import Text "mo:base/Text"; + +import T "Types"; +import U "Utils"; + +module { + public type AssetEncoding = { + contentEncoding: Text; + content: [Blob]; + totalLength: Nat; + }; + public class Asset( + initContentType: Text, + initEncodings: HashMap.HashMap + ) { + public let contentType = initContentType; + public let encodings = initEncodings; + + public func getEncoding(acceptEncodings : [Text]) : ?AssetEncoding { + for (acceptEncoding in acceptEncodings.vals()) { + switch (encodings.get(acceptEncoding)) { + case null {}; + case (?encoding) return ?encoding; + } + }; + null + }; + }; + /* + //public type Asset = { + // contentType: Text; + // encodings: HashMap.HashMap; + //}; + + public func getAssetEncoding(asset : Asset, acceptEncodings : [Text]) : ?AssetEncoding { + for (acceptEncoding in acceptEncodings.vals()) { + switch (asset.encodings.get(acceptEncoding)) { + case null {}; + case (?encoding) return ?encoding; + } + }; + null + };*/ + + public type StableAsset = { + contentType: Text; + encodings: [(Text, AssetEncoding)]; + }; + + public func toStableAssetEntry((k: T.Key, v: Asset)) : ((T.Key, StableAsset)) { + let sa : StableAsset = { + contentType = v.contentType; + encodings = Iter.toArray(v.encodings.entries()); + }; + (k, sa) + }; + + public func toAssetEntry((k: T.Key, v: StableAsset)) : ((T.Key, Asset)) { + let a = Asset( + v.contentType, + HashMap.fromIter(v.encodings.vals(), 7, Text.equal, Text.hash) + ); + (k, a) + }; + +} \ No newline at end of file diff --git a/src/distributed/assetstorage/Batch.mo b/src/distributed/assetstorage/Batch.mo new file mode 100644 index 0000000000..aebb3479ad --- /dev/null +++ b/src/distributed/assetstorage/Batch.mo @@ -0,0 +1,69 @@ +import Debug "mo:base/Debug"; +import Int "mo:base/Int"; +import Time "mo:base/Time"; + +import H "mo:base/HashMap"; +import T "Types"; +import U "Utils"; + +module { + +object batch { + public func nextExpireTime() : T.Time { + let expiryNanos = 5 * 60 * 1000 * 1000 * 1000; + Time.now() + expiryNanos + } +}; + +// A batch associates a bunch of chunks that are being uploaded, so that none +// of them time out or all of them do. +public class Batch() { + var expiresAt : T.Time = batch.nextExpireTime(); + + public func refreshExpiry() { + expiresAt := batch.nextExpireTime(); + }; + + public func isExpired(asOf : T.Time) : Bool { + expiresAt <= asOf + }; +}; + +// We group the staged chunks into batches. Uploading a chunk refreshes the batch's expiry timer. +// We delete expired batches so that they don't consume space forever after an interrupted install. +public class Batches() { + var nextBatchId = 1; + let batches = H.HashMap(7, Int.equal, Int.hash); + + public func get(batchId: T.BatchId) : ?Batch { + batches.get(batchId) + }; + + public func delete(batchId: T.BatchId) { + batches.delete(batchId) + }; + + public func startBatch(): T.BatchId { + let batch_id = nextBatchId; + nextBatchId += 1; + let batch = Batch(); + batches.put(batch_id, batch); + batch_id + }; + + public func deleteExpired() : () { + let now = Time.now(); + U.deleteFromHashMap(batches, Int.equal, Int.hash, + func(k: Int, batch: Batch) : Bool { + batch.isExpired(now) + } + ); + }; + + public func reset() { + nextBatchId := 1; + U.clearHashMap(batches); + } +} + +} \ No newline at end of file diff --git a/src/distributed/assetstorage/Batches.mo b/src/distributed/assetstorage/Batches.mo deleted file mode 100644 index b652db5b24..0000000000 --- a/src/distributed/assetstorage/Batches.mo +++ /dev/null @@ -1,60 +0,0 @@ -import Debug "mo:base/Debug"; -import Int "mo:base/Int"; -import Time "mo:base/Time"; - -import H "mo:base/HashMap"; -import T "Types"; -import U "Utils"; - -module { - -public class Batches() { - // We group the staged chunks into batches. Uploading a chunk refreshes the batch's expiry timer. - // We delete expired batches so that they don't consume space after an interrupted install. - var nextBatchId = 1; - let batches = H.HashMap(7, Int.equal, Int.hash); - - public func get(batchId: T.BatchId) : ?T.Batch { - batches.get(batchId) - }; - public func delete(batchId: T.BatchId) { - batches.delete(batchId) - }; - - public func startBatch(): T.BatchId { - let batch_id = nextBatchId; - nextBatchId += 1; - let batch = T.Batch(); - batches.put(batch_id, batch); - batch_id - }; - - public func deleteExpired() : () { - let now = Time.now(); - /*let batchesToDelete: H.HashMap = H.mapFilter(batches, Int.equal, Int.hash, - func(k: Int, batch: T.Batch) : ?T.Batch { - if (batch.expired(now)) - ?batch - else - null - } - ); - for ((k,_) in batchesToDelete.entries()) { - Debug.print("delete expired batch " # Int.toText(k)); - - batches.delete(k); - };*/ - U.deleteFromHashMap(batches, Int.equal, Int.hash, - func(k: Int, batch: T.Batch) : Bool { - batch.expired(now) - } - ); - }; - - public func reset() { - nextBatchId := 1; - U.clearHashMap(batches); - } -} - -} \ No newline at end of file diff --git a/src/distributed/assetstorage/Chunk.mo b/src/distributed/assetstorage/Chunk.mo new file mode 100644 index 0000000000..827e299258 --- /dev/null +++ b/src/distributed/assetstorage/Chunk.mo @@ -0,0 +1,63 @@ +import Debug "mo:base/Debug"; +import HashMap "mo:base/HashMap"; +import Int "mo:base/Int"; +import Result "mo:base/Result"; +import Time "mo:base/Time"; + +import B "Batch"; +import T "Types"; +import U "Utils"; + +module Chunk { + +// A chunks holds a staged piece of content until we assign it to +// an asset by content-encoding. +public type Chunk = { + batch: B.Batch; + content: Blob; +}; + +public class Chunks() { + var nextChunkId = 1; + let chunks = HashMap.HashMap(7, Int.equal, Int.hash); + + // Create a new chunk for a piece of content. This refreshes the batch's + // expiry timer. + public func create(batch: B.Batch, content: Blob) : T.ChunkId { + let chunkId = nextChunkId; + nextChunkId += 1; + let chunk : Chunk = { + batch = batch; + content = content; + }; + + batch.refreshExpiry(); + chunks.put(chunkId, chunk); + + chunkId + }; + + public func take(chunkId: T.ChunkId): Result.Result { + switch (chunks.remove(chunkId)) { + case null #err("chunk not found"); + case (?chunk) #ok(chunk.content); + } + }; + + public func reset() { + nextChunkId := 1; + U.clearHashMap(chunks); + }; + + public func deleteExpired() : () { + let now = Time.now(); + + U.deleteFromHashMap(chunks, Int.equal, Int.hash, + func(k: Int, chunk: Chunk) : Bool { + chunk.batch.isExpired(now) + } + ); + }; +} + +} \ No newline at end of file diff --git a/src/distributed/assetstorage/Chunks.mo b/src/distributed/assetstorage/Chunks.mo deleted file mode 100644 index 4db63934db..0000000000 --- a/src/distributed/assetstorage/Chunks.mo +++ /dev/null @@ -1,67 +0,0 @@ -import Debug "mo:base/Debug"; -import HashMap "mo:base/HashMap"; -import Int "mo:base/Int"; -import Result "mo:base/Result"; -import Time "mo:base/Time"; - -import T "Types"; -import U "Utils"; - -module { - -public class Chunks() { - // We stage asset content chunks here, - // before assigning them to asset content encodings. - var nextChunkId = 1; - let chunks = HashMap.HashMap(7, Int.equal, Int.hash); - - public func create(batch: T.Batch, content: Blob) : T.ChunkId { - let chunkId = nextChunkId; - nextChunkId += 1; - let chunk : T.Chunk = { - batch = batch; - content = content; - }; - - batch.refreshExpiry(); - chunks.put(chunkId, chunk); - - chunkId - }; - - public func take(chunkId: T.ChunkId): Result.Result { - switch (chunks.remove(chunkId)) { - case null #err("chunk not found"); - case (?chunk) #ok(chunk.content); - } - }; - - public func reset() { - nextChunkId := 1; - U.clearHashMap(chunks); - }; - - public func deleteExpired() : () { - let now = Time.now(); - /*let chunksToDelete = HashMap.mapFilter(chunks, Int.equal, Int.hash, - func(k: Int, chunk: T.Chunk) : ?T.Chunk { - if (chunk.batch.expired(now)) - ?chunk - else - null - } - ); - for ((k,_) in chunksToDelete.entries()) { - Debug.print("delete expired chunk " # Int.toText(k)); - chunks.delete(k); - };*/ - U.deleteFromHashMap(chunks, Int.equal, Int.hash, - func(k: Int, chunk: T.Chunk) : Bool { - chunk.batch.expired(now) - } - ); - - }; -} - -} \ No newline at end of file diff --git a/src/distributed/assetstorage/Main.mo b/src/distributed/assetstorage/Main.mo index cc6c70cce0..0942df3d99 100644 --- a/src/distributed/assetstorage/Main.mo +++ b/src/distributed/assetstorage/Main.mo @@ -10,8 +10,9 @@ import Result "mo:base/Result"; import Text "mo:base/Text"; import Time "mo:base/Time"; import Tree "mo:base/RBTree"; -import B "Batches"; -import C "Chunks"; +import A "Asset"; +import B "Batch"; +import C "Chunk"; import T "Types"; import U "Utils"; @@ -20,31 +21,20 @@ shared ({caller = creator}) actor class () { stable var authorized: [Principal] = [creator]; - stable var stableAssets : [(T.Key, T.StableAsset)] = []; - let assets = HashMap.fromIter(Iter.map(stableAssets.vals(), T.fromStableAssetEntry), 7, Text.equal, Text.hash); + stable var stableAssets : [(T.Key, A.StableAsset)] = []; + let assets = HashMap.fromIter(Iter.map(stableAssets.vals(), A.toAssetEntry), 7, Text.equal, Text.hash); let chunks = C.Chunks(); let batches = B.Batches(); system func preupgrade() { - stableAssets := Iter.toArray(Iter.map(assets.entries(), T.fromAssetEntry)); + stableAssets := Iter.toArray(Iter.map(assets.entries(), A.toStableAssetEntry)); }; system func postupgrade() { stableAssets := []; }; - func getAssetEncoding(asset : T.Asset, acceptEncodings : [Text]) : ?T.AssetEncoding { - for (acceptEncoding in acceptEncodings.vals()) { - switch (asset.encodings.get(acceptEncoding)) { - case null {}; - case (?encoding) return ?encoding; - } - }; - null - }; - - public shared ({ caller }) func authorize(other: Principal) : async () { if (isSafe(caller)) { authorized := Array.append(authorized, [other]); @@ -88,7 +78,7 @@ shared ({caller = creator}) actor class () { switch (assets.get(path)) { case null throw Error.reject("not found"); case (?asset) { - switch (getAssetEncoding(asset, ["identity"])) { + switch (asset.getEncoding(["identity"])) { case null throw Error.reject("no such encoding"); case (?encoding) { encoding.content[0] @@ -99,7 +89,7 @@ shared ({caller = creator}) actor class () { }; public query func list() : async [T.Path] { - let iter = Iter.map<(Text, T.Asset), T.Path>(assets.entries(), func (key, _) = key); + let iter = Iter.map<(Text, A.Asset), T.Path>(assets.entries(), func (key, _) = key); Iter.toArray(iter) }; @@ -121,7 +111,7 @@ shared ({caller = creator}) actor class () { switch (assets.get(arg.key)) { case null throw Error.reject("asset not found"); case (?asset) { - switch (getAssetEncoding(asset, arg.accept_encodings)) { + switch (asset.getEncoding(arg.accept_encodings)) { case null throw Error.reject("no such encoding"); case (?encoding) { { @@ -206,7 +196,7 @@ shared ({caller = creator}) actor class () { case (#SetAssetContent(args)) { setAssetContent(args); }; case (#UnsetAssetContent(args)) { unsetAssetContent(args); }; case (#DeleteAsset(args)) { deleteAsset(args); }; - case (#Clear(args)) { clearEverything(args); } + case (#Clear(args)) { doClear(args); } }; switch(r) { case (#ok(())) {}; @@ -230,10 +220,10 @@ shared ({caller = creator}) actor class () { Debug.print("createAsset(" # arg.key # ")"); switch (assets.get(arg.key)) { case null { - let asset : T.Asset = { - contentType = arg.content_type; - encodings = HashMap.HashMap(7, Text.equal, Text.hash); - }; + let asset = A.Asset( + arg.content_type, + HashMap.HashMap(7, Text.equal, Text.hash) + ); assets.put(arg.key, asset ); }; case (?asset) { @@ -271,13 +261,6 @@ shared ({caller = creator}) actor class () { return false; } }; - //var i = 1; - //var last = chunks.size() - 1; - //while (i <= last) { - // if (chunks[i].size() != expectedLength) - // return false; - // i += 1; - //}; true }; @@ -291,7 +274,7 @@ shared ({caller = creator}) actor class () { if (chunkLengthsMatch(chunks) == false) { #err("chunk lengths do not match the size of the first chunk") } else { - let encoding : T.AssetEncoding = { + let encoding : A.AssetEncoding = { contentEncoding = arg.content_encoding; content = chunks; totalLength = Array.foldLeft(chunks, 0, addBlobLength); @@ -318,7 +301,14 @@ shared ({caller = creator}) actor class () { }; func unsetAssetContent(args: T.UnsetAssetContentArguments) : Result.Result<(), Text> { - #err("unset_asset_content: not implemented"); + Debug.print("unsetAssetContent(" # args.key # ")"); + switch (assets.get(args.key)) { + case null #err("asset not found"); + case (?asset) { + asset.encodings.delete(args.content_encoding); + #ok(()) + }; + }; }; public shared ({ caller }) func delete_asset(args: T.DeleteAssetArguments) : async () { @@ -343,13 +333,13 @@ shared ({caller = creator}) actor class () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - switch(clearEverything(args)) { + switch(doClear(args)) { case (#ok(())) {}; case (#err(err)) throw Error.reject(err); }; }; - func clearEverything(args: T.ClearArguments) : Result.Result<(), Text> { + func doClear(args: T.ClearArguments) : Result.Result<(), Text> { stableAssets := []; U.clearHashMap(assets); @@ -359,7 +349,7 @@ shared ({caller = creator}) actor class () { #ok(()) }; - public func version_14() : async() { + public func version_15() : async() { }; }; diff --git a/src/distributed/assetstorage/StableHashMap.mo b/src/distributed/assetstorage/StableHashMap.mo deleted file mode 100644 index 81d8990bc2..0000000000 --- a/src/distributed/assetstorage/StableHashMap.mo +++ /dev/null @@ -1,142 +0,0 @@ -import Prim "mo:prim"; -import P "mo:base/Prelude"; -import A "mo:base/Array"; -import Hash "mo:base/Hash"; -import Iter "mo:base/Iter"; -import AssocList "mo:base/AssocList"; - -module { - -// key-val list type -type KVs = AssocList.AssocList; - -public class StableHashMap() { - public var table : [var KVs] = [var]; - public var _count : Nat = 0; -}; - -public class StableHashMapManipulator( - initCapacity: Nat, - keyEq: (K,K) -> Bool, - keyHash: K -> Hash.Hash -) { - - /// Returns the number of entries in this HashMap. - public func size(shm: StableHashMap) : Nat = shm._count; - - /// Deletes the entry with the key `k`. Doesn't do anything if the key doesn't - /// exist. - public func delete(shm: StableHashMap, k : K) = ignore remove(shm, k); - - /// Removes the entry with the key `k` and returns the associated value if it - /// existed or `null` otherwise. - public func remove(shm: StableHashMap, k : K) : ?V { - let h = Prim.word32ToNat(keyHash(k)); - let m = shm.table.size(); - if (m > 0) { - let pos = h % m; - let (kvs2, ov) = AssocList.replace(shm.table[pos], k, keyEq, null); - shm.table[pos] := kvs2; - switch(ov){ - case null { }; - case _ { shm._count -= 1; } - }; - ov - } else { - null - }; - }; - - - /// Gets the entry with the key `k` and returns its associated value if it - /// existed or `null` otherwise. - public func get(shm: StableHashMap, k:K) : ?V { - let h = Prim.word32ToNat(keyHash(k)); - let m = shm.table.size(); - let v = if (m > 0) { - AssocList.find(shm.table[h % m], k, keyEq) - } else { - null - }; - }; - - /// Insert the value `v` at key `k`. Overwrites an existing entry with key `k` - public func put(m: StableHashMap, k : K, v : V) = ignore replace(m, k, v); - - /// Insert the value `v` at key `k` and returns the previous value stored at - /// `k` or null if it didn't exist. - public func replace(m: StableHashMap, k:K, v:V) : ?V { - if (m._count >= m.table.size()) { - let size = - if (m._count == 0) { - if (initCapacity > 0) { - initCapacity - } else { - 1 - } - } else { - m.table.size() * 2; - }; - let table2 = A.init>(size, null); - for (i in m.table.keys()) { - var kvs = m.table[i]; - label moveKeyVals : () - loop { - switch kvs { - case null { break moveKeyVals }; - case (?((k, v), kvsTail)) { - let h = Prim.word32ToNat(keyHash(k)); - let pos2 = h % table2.size(); - table2[pos2] := ?((k,v), table2[pos2]); - kvs := kvsTail; - }; - } - }; - }; - m.table := table2; - }; - let h = Prim.word32ToNat(keyHash(k)); - let pos = h % m.table.size(); - let (kvs2, ov) = AssocList.replace(m.table[pos], k, keyEq, ?v); - m.table[pos] := kvs2; - switch(ov){ - case null { m._count += 1 }; - case _ {} - }; - ov - }; - - /// Returns an iterator over the key value pairs in this - /// HashMap. Does _not_ modify the HashMap. - public func entries(m: StableHashMap) : Iter.Iter<(K,V)> { - if (m.table.size() == 0) { - object { public func next() : ?(K,V) { null } } - } - else { - object { - var kvs = m.table[0]; - var nextTablePos = 1; - public func next () : ?(K,V) { - switch kvs { - case (?(kv, kvs2)) { - kvs := kvs2; - ?kv - }; - case null { - if (nextTablePos < m.table.size()) { - kvs := m.table[nextTablePos]; - nextTablePos += 1; - next() - } else { - null - } - } - } - }; - } - } - }; - -}; - -} \ No newline at end of file diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo index 2993f83188..e377344c1d 100644 --- a/src/distributed/assetstorage/Types.mo +++ b/src/distributed/assetstorage/Types.mo @@ -1,22 +1,11 @@ -import H "mo:base/HashMap"; -import Iter "mo:base/Iter"; -import Text "mo:base/Text"; -import Time "mo:base/Time"; - module Types { public type Contents = Blob; public type Path = Text; public type BatchId = Nat; - public type BlobId = Text; public type ChunkId = Nat; - public type ContentEncoding = Text; - public type ContentType = Text; - public type EncodingId = Text; public type Key = Text; - public type Offset = Nat; public type Time = Int; - public type TotalLength = Nat; public type CreateAssetArguments = { key: Key; @@ -55,61 +44,4 @@ module Types { batch_id: BatchId; operations: [BatchOperationKind]; }; - - public type AssetEncoding = { - contentEncoding: Text; - content: [Blob]; - totalLength: Nat; - }; - - public type Asset = { - contentType: Text; - encodings: H.HashMap; - }; - - public type StableAsset = { - contentType: Text; - encodings: [(Text, AssetEncoding)]; - }; - - public func fromAssetEntry((k: Key, v: Asset)) : ((Key, StableAsset)) { - let fa : StableAsset = { - contentType = v.contentType; - encodings = Iter.toArray(v.encodings.entries()); - }; - (k, fa) - }; - - public func fromStableAssetEntry((k: Key, v: StableAsset)) : ((Key, Asset)) { - let a : Asset = { - contentType = v.contentType; - encodings = H.fromIter(v.encodings.vals(), 7, Text.equal, Text.hash); - }; - (k, a) - }; - - public type Chunk = { - batch: Batch; - content: Blob; - }; - - object batch { - let expiryNanos = 300_000_000_000; // 5 * 60 * 1000 * 1000 * 1000; - - public func nextExpireTime() : Time { - Time.now() + expiryNanos - } - }; - - public class Batch() { - var expiresAt : Time = batch.nextExpireTime(); - - public func refreshExpiry() { - expiresAt := batch.nextExpireTime(); - }; - - public func expired(asOf : Time) : Bool { - expiresAt <= asOf - }; - }; }; \ No newline at end of file From c24f2a10043a2418c65aa159edc2473b124ed4c4 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 8 Mar 2021 11:55:24 -0800 Subject: [PATCH 38/48] wip --- src/distributed/assetstorage/Asset.mo | 42 +++++----- src/distributed/assetstorage/Batch.mo | 19 ++--- src/distributed/assetstorage/Chunk.mo | 6 +- src/distributed/assetstorage/Main.mo | 110 +++++++++++++------------- src/distributed/assetstorage/Utils.mo | 34 ++++---- 5 files changed, 103 insertions(+), 108 deletions(-) diff --git a/src/distributed/assetstorage/Asset.mo b/src/distributed/assetstorage/Asset.mo index cc0dbe7797..f32d91e1a3 100644 --- a/src/distributed/assetstorage/Asset.mo +++ b/src/distributed/assetstorage/Asset.mo @@ -11,14 +11,15 @@ module { content: [Blob]; totalLength: Nat; }; + public class Asset( initContentType: Text, initEncodings: HashMap.HashMap ) { public let contentType = initContentType; - public let encodings = initEncodings; + let encodings = initEncodings; - public func getEncoding(acceptEncodings : [Text]) : ?AssetEncoding { + public func chooseEncoding(acceptEncodings : [Text]) : ?AssetEncoding { for (acceptEncoding in acceptEncodings.vals()) { switch (encodings.get(acceptEncoding)) { case null {}; @@ -27,22 +28,26 @@ module { }; null }; - }; - /* - //public type Asset = { - // contentType: Text; - // encodings: HashMap.HashMap; - //}; - public func getAssetEncoding(asset : Asset, acceptEncodings : [Text]) : ?AssetEncoding { - for (acceptEncoding in acceptEncodings.vals()) { - switch (asset.encodings.get(acceptEncoding)) { - case null {}; - case (?encoding) return ?encoding; + public func getEncoding(encodingType: Text): ?AssetEncoding { + encodings.get(encodingType) + }; + + public func setEncoding(encodingType: Text, encoding: AssetEncoding) { + encodings.put(encodingType, encoding) + }; + + public func unsetEncoding(encodingType: Text) { + encodings.delete(encodingType) + }; + + public func toStableAsset() : StableAsset { + { + contentType = contentType; + encodings = Iter.toArray(encodings.entries()); } }; - null - };*/ + }; public type StableAsset = { contentType: Text; @@ -50,11 +55,7 @@ module { }; public func toStableAssetEntry((k: T.Key, v: Asset)) : ((T.Key, StableAsset)) { - let sa : StableAsset = { - contentType = v.contentType; - encodings = Iter.toArray(v.encodings.entries()); - }; - (k, sa) + (k, v.toStableAsset()) }; public func toAssetEntry((k: T.Key, v: StableAsset)) : ((T.Key, Asset)) { @@ -64,5 +65,4 @@ module { ); (k, a) }; - } \ No newline at end of file diff --git a/src/distributed/assetstorage/Batch.mo b/src/distributed/assetstorage/Batch.mo index aebb3479ad..e0871a0bce 100644 --- a/src/distributed/assetstorage/Batch.mo +++ b/src/distributed/assetstorage/Batch.mo @@ -17,7 +17,8 @@ object batch { // A batch associates a bunch of chunks that are being uploaded, so that none // of them time out or all of them do. -public class Batch() { +public class Batch(initBatchId: T.BatchId) { + public let batchId = initBatchId; var expiresAt : T.Time = batch.nextExpireTime(); public func refreshExpiry() { @@ -43,21 +44,17 @@ public class Batches() { batches.delete(batchId) }; - public func startBatch(): T.BatchId { - let batch_id = nextBatchId; + public func startBatch(): Batch { + let batchId = nextBatchId; nextBatchId += 1; - let batch = Batch(); - batches.put(batch_id, batch); - batch_id + let batch = Batch(batchId); + batches.put(batchId, batch); + batch }; public func deleteExpired() : () { let now = Time.now(); - U.deleteFromHashMap(batches, Int.equal, Int.hash, - func(k: Int, batch: Batch) : Bool { - batch.isExpired(now) - } - ); + U.deleteFromHashMap(batches, Int.equal, Int.hash, func(k: Int, batch: Batch) : Bool = batch.isExpired(now)); }; public func reset() { diff --git a/src/distributed/assetstorage/Chunk.mo b/src/distributed/assetstorage/Chunk.mo index 827e299258..de563ddc6c 100644 --- a/src/distributed/assetstorage/Chunk.mo +++ b/src/distributed/assetstorage/Chunk.mo @@ -52,11 +52,7 @@ public class Chunks() { public func deleteExpired() : () { let now = Time.now(); - U.deleteFromHashMap(chunks, Int.equal, Int.hash, - func(k: Int, chunk: Chunk) : Bool { - chunk.batch.isExpired(now) - } - ); + U.deleteFromHashMap(chunks, Int.equal, Int.hash, func(k: Int, chunk: Chunk) : Bool = chunk.batch.isExpired(now)); }; } diff --git a/src/distributed/assetstorage/Main.mo b/src/distributed/assetstorage/Main.mo index 0942df3d99..987da30c05 100644 --- a/src/distributed/assetstorage/Main.mo +++ b/src/distributed/assetstorage/Main.mo @@ -9,7 +9,6 @@ import Nat8 "mo:base/Nat8"; import Result "mo:base/Result"; import Text "mo:base/Text"; import Time "mo:base/Time"; -import Tree "mo:base/RBTree"; import A "Asset"; import B "Batch"; import C "Chunk"; @@ -43,16 +42,34 @@ shared ({caller = creator}) actor class () { } }; + // Retrieve an asset's contents by name. Only returns the first chunk of an asset's + // contents, even if there were more than one chunk. + // To handle larger assets, use get() and get_chunk(). + public query func retrieve(path : T.Path) : async T.Contents { + switch (assets.get(path)) { + case null throw Error.reject("not found"); + case (?asset) { + switch (asset.getEncoding("identity")) { + case null throw Error.reject("no identity encoding"); + case (?encoding) encoding.content[0]; + }; + }; + } + }; + + // Store an asset of limited size, with + // content-type: "application/octet-stream" + // content-encoding: "identity" + // This deprecated function is backwards-compatible with an older interface and will be replaced + // with a function of the same name but that allows specification of the content type and encoding. + // Prefer to use create_batch(), create_chunk(), commit_batch(). public shared ({ caller }) func store(path : T.Path, contents : T.Contents) : async () { if (isSafe(caller) == false) { throw Error.reject("not authorized"); }; - let batch_id = batches.startBatch(); // ew - let chunk_id = switch (batches.get(batch_id)) { - case null throw Error.reject("batch not found"); - case (?batch) chunks.create(batch, contents) - }; + let batch = batches.startBatch(); + let chunkId = chunks.create(batch, contents); let create_asset_args : T.CreateAssetArguments = { key = path; @@ -63,42 +80,31 @@ shared ({caller = creator}) actor class () { case (#err(msg)) throw Error.reject(msg); }; - let set_asset_content_args : T.SetAssetContentArguments = { + let args : T.SetAssetContentArguments = { key = path; content_encoding = "identity"; - chunk_ids = [ chunk_id ]; + chunk_ids = [ chunkId ]; }; - switch(setAssetContent(set_asset_content_args)) { + switch(setAssetContent(args)) { case (#ok(())) {}; case (#err(msg)) throw Error.reject(msg); }; }; - public query func retrieve(path : T.Path) : async T.Contents { - switch (assets.get(path)) { - case null throw Error.reject("not found"); - case (?asset) { - switch (asset.getEncoding(["identity"])) { - case null throw Error.reject("no such encoding"); - case (?encoding) { - encoding.content[0] - } - }; - }; - } - }; - public query func list() : async [T.Path] { let iter = Iter.map<(Text, A.Asset), T.Path>(assets.entries(), func (key, _) = key); Iter.toArray(iter) }; func isSafe(caller: Principal) : Bool { - return true; func eq(value: Principal): Bool = value == caller; Array.find(authorized, eq) != null }; + // Choose a content encoding from among the accepted encodings and return + // its content, or the first chunk of its content. + // If content.size() > total_length, call get_chunk() get the rest of the content. + // All chunks except the last will have the same size as the first chunk. public query func get(arg:{ key: T.Key; accept_encodings: [Text] @@ -111,7 +117,7 @@ shared ({caller = creator}) actor class () { switch (assets.get(arg.key)) { case null throw Error.reject("asset not found"); case (?asset) { - switch (asset.getEncoding(arg.accept_encodings)) { + switch (asset.chooseEncoding(arg.accept_encodings)) { case null throw Error.reject("no such encoding"); case (?encoding) { { @@ -126,6 +132,7 @@ shared ({caller = creator}) actor class () { }; }; + // Get subsequent chunks of an asset encoding's content, after get(). public query func get_chunk(arg:{ key: T.Key; content_encoding: Text; @@ -136,7 +143,7 @@ shared ({caller = creator}) actor class () { switch (assets.get(arg.key)) { case null throw Error.reject("asset not found"); case (?asset) { - switch (asset.encodings.get(arg.content_encoding)) { + switch (asset.getEncoding(arg.content_encoding)) { case null throw Error.reject("no such encoding"); case (?encoding) { { @@ -148,6 +155,7 @@ shared ({caller = creator}) actor class () { }; }; + // All chunks are associated with a batch until committed with commit_batch. public shared ({ caller }) func create_batch(arg: {}) : async ({ batch_id: T.BatchId }) { @@ -160,7 +168,7 @@ shared ({caller = creator}) actor class () { chunks.deleteExpired(); { - batch_id = batches.startBatch(); + batch_id = batches.startBatch().batchId; } }; @@ -174,13 +182,13 @@ shared ({caller = creator}) actor class () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - switch (batches.get(arg.batch_id)) { + let chunkId = switch (batches.get(arg.batch_id)) { case null throw Error.reject("batch not found"); - case (?batch) { - { - chunk_id = chunks.create(batch, arg.content) - } - } + case (?batch) chunks.create(batch, arg.content) + }; + + { + chunk_id = chunkId; } }; @@ -190,7 +198,6 @@ shared ({caller = creator}) actor class () { throw Error.reject("not authorized"); for (op in args.operations.vals()) { - let r : Result.Result<(), Text> = switch(op) { case (#CreateAsset(args)) { createAsset(args); }; case (#SetAssetContent(args)) { setAssetContent(args); }; @@ -244,22 +251,17 @@ shared ({caller = creator}) actor class () { }; }; - func addBlobLength(acc: Nat, blob: Blob): Nat { - acc + blob.size() - }; - func chunkLengthsMatch(chunks: [Blob]): Bool { - if (chunks.size() == 0) - return true; - - let expectedLength = chunks[0].size(); - for (i in Iter.range(1, chunks.size()-2)) { - Debug.print("chunk at index " # Int.toText(i) # " has length " # Int.toText(chunks[i].size()) # " and expected is " # Int.toText(expectedLength) ); - if (chunks[i].size() != expectedLength) { - Debug.print("chunk at index " # Int.toText(i) # " with length " # Int.toText(chunks[i].size()) # " does not match expected length " # Int.toText(expectedLength) ); - - return false; - } + if (chunks.size() > 2) { + let expectedLength = chunks[0].size(); + for (i in Iter.range(1, chunks.size()-2)) { + Debug.print("chunk at index " # Int.toText(i) # " has length " # Int.toText(chunks[i].size()) # " and expected is " # Int.toText(expectedLength) ); + if (chunks[i].size() != expectedLength) { + Debug.print("chunk at index " # Int.toText(i) # " with length " # Int.toText(chunks[i].size()) # " does not match expected length " # Int.toText(expectedLength) ); + + return false; + } + }; }; true }; @@ -277,11 +279,11 @@ shared ({caller = creator}) actor class () { let encoding : A.AssetEncoding = { contentEncoding = arg.content_encoding; content = chunks; - totalLength = Array.foldLeft(chunks, 0, addBlobLength); + totalLength = Array.foldLeft(chunks, 0, func (acc: Nat, blob: Blob): Nat { + acc + blob.size() + }); }; - - asset.encodings.put(arg.content_encoding, encoding); - #ok(()); + #ok(asset.setEncoding(arg.content_encoding, encoding)); }; }; case (#err(err)) #err(err); @@ -305,7 +307,7 @@ shared ({caller = creator}) actor class () { switch (assets.get(args.key)) { case null #err("asset not found"); case (?asset) { - asset.encodings.delete(args.content_encoding); + asset.unsetEncoding(args.content_encoding); #ok(()) }; }; diff --git a/src/distributed/assetstorage/Utils.mo b/src/distributed/assetstorage/Utils.mo index ec7238d112..89f681b020 100644 --- a/src/distributed/assetstorage/Utils.mo +++ b/src/distributed/assetstorage/Utils.mo @@ -6,8 +6,8 @@ import Iter "mo:base/Iter"; module Utils { - public func clearHashMap(h:HashMap.HashMap) : () { - let keys = Iter.toArray(Iter.map(h.entries(), func((k: K, v: V)): K { k })); + public func clearHashMap(h:HashMap.HashMap) : () { + let keys = Iter.toArray(Iter.map(h.entries(), func((k: K, _: V)): K = k )); for (key in keys.vals()) { h.delete(key); }; @@ -17,19 +17,19 @@ module Utils { (h:HashMap.HashMap, keyEq: (K,K) -> Bool, keyHash: K -> Hash.Hash, - deleteFn: (K, V) -> Bool): () { - let entriesToDelete: HashMap.HashMap = HashMap.mapFilter(h, keyEq, keyHash, - func(k: K, v: V) : ?V { - if (deleteFn(k, v)) - ?v - else - null - } - ); - for ((k,_) in entriesToDelete.entries()) { - Debug.print("delete expired"); - h.delete(k); - }; - + deleteFn: (K, V) -> Bool + ): () { + let entriesToDelete: HashMap.HashMap = HashMap.mapFilter(h, keyEq, keyHash, + func(k: K, v: V) : ?V { + if (deleteFn(k, v)) + ?v + else + null + } + ); + for ((k,_) in entriesToDelete.entries()) { + Debug.print("delete expired"); + h.delete(k); + }; } -}; \ No newline at end of file +}; From 83630e6f36a9cc6c46a8e0cb3addece687b5851a Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 8 Mar 2021 15:19:02 -0800 Subject: [PATCH 39/48] cleanup --- e2e/tests-dfx/assetscanister.bash | 4 +- src/dfx/src/lib/installers/assets.rs | 81 +++++++++------------------ src/distributed/assetstorage/Asset.mo | 11 ++-- src/distributed/assetstorage/Batch.mo | 8 +-- src/distributed/assetstorage/Chunk.mo | 2 +- src/distributed/assetstorage/Main.mo | 34 +++++------ src/distributed/assetstorage/Types.mo | 2 +- src/distributed/assetstorage/Utils.mo | 2 - 8 files changed, 52 insertions(+), 92 deletions(-) diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index 05f5de6298..ff9003193b 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -14,7 +14,6 @@ teardown() { } @test "can store and retrieve assets by key" { - skip install_asset assetscanister dfx_start @@ -68,5 +67,4 @@ teardown() { assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="large-asset.bin";content_encoding="identity";index=13})' assert_command_fail dfx canister call --query e2e_project_assets get_chunk '(record{key="large-asset.bin";content_encoding="identity";index=14})' - -} \ No newline at end of file +} diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index fe62d03798..0958174068 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -5,16 +5,13 @@ use crate::lib::waiter::waiter_with_timeout; use candid::{CandidType, Decode, Encode}; use delay::{Delay, Waiter}; -//use futures::future::try_join_all; use ic_agent::Agent; use ic_types::Principal; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::time::Duration; -//use tokio::task; use walkdir::WalkDir; -//const GET: &str = "get"; const CREATE_BATCH: &str = "create_batch"; const CREATE_CHUNK: &str = "create_chunk"; const COMMIT_BATCH: &str = "commit_batch"; @@ -114,7 +111,6 @@ async fn create_chunk( ) -> DfxResult { let args = CreateChunkRequest { batch_id, content }; let args = candid::Encode!(&args)?; - println!("create chunk"); let mut waiter = Delay::builder() .timeout(std::time::Duration::from_secs(30)) @@ -136,18 +132,12 @@ async fn create_chunk( .map(|x| x.chunk_id) }) { Ok(chunk_id) => { - println!("created chunk {}", chunk_id); break Ok(chunk_id); } - Err(agent_err) => { - println!("agent error ({}) waiting to retry...", agent_err); - match waiter.wait() { - Ok(()) => { - println!("retrying..."); - } - Err(_) => break Err(agent_err), - } - } + Err(agent_err) => match waiter.wait() { + Ok(()) => {} + Err(_) => break Err(agent_err), + }, } } } @@ -160,12 +150,8 @@ async fn make_chunked_asset( asset_location: AssetLocation, ) -> DfxResult { let content = &std::fs::read(&asset_location.source)?; - println!( - "create chunks for {}", - asset_location.source.to_string_lossy() - ); - // ?? doesn't work + // ?? doesn't work: rust lifetimes + task::spawn = tears // how to deal with lifetimes for agent and canister_id here // this function won't exit until after the task is joined... // let chunks_future_tasks: Vec<_> = content @@ -183,7 +169,8 @@ async fn make_chunked_asset( // }); // ?? doesn't work - // works (sometimes) + // works (sometimes), does more work concurrently, but often doesn't work against bootstrap. + // (connection stuck in odd idle state: all agent requests return "channel closed" error.) // let chunks_futures: Vec<_> = content // .chunks(MAX_CHUNK_SIZE) // .map(|content| create_chunk(agent, canister_id, timeout, batch_id, content)) @@ -199,7 +186,18 @@ async fn make_chunked_asset( // works (sometimes) let mut chunk_ids: Vec = vec![]; - for data_chunk in content.chunks(MAX_CHUNK_SIZE) { + let chunks = content.chunks(MAX_CHUNK_SIZE); + let (num_chunks, _) = chunks.size_hint(); + let mut i: usize = 0; + for data_chunk in chunks { + i += 1; + println!( + " {} {}/{} ({} bytes)", + &asset_location.relative.to_string_lossy(), + i, + num_chunks, + data_chunk.len() + ); chunk_ids.push(create_chunk(agent, canister_id, timeout, batch_id, data_chunk).await?); } Ok(ChunkedAsset { @@ -215,6 +213,8 @@ async fn make_chunked_assets( batch_id: u128, locs: Vec, ) -> DfxResult> { + // this neat futures version works faster in parallel when it works, + // but does not work often when connecting through the bootstrap. // let futs: Vec<_> = locs // .into_iter() // .map(|loc| make_chunked_asset(agent, canister_id, timeout, batch_id, loc)) @@ -262,16 +262,12 @@ async fn commit_batch( operations, }; let arg = candid::Encode!(&arg)?; - println!("encoded arg: {:02X?}", arg); - let idl = candid::IDLArgs::from_bytes(&arg)?; - println!("{:?}", idl); - let x = agent + agent .update(&canister_id, COMMIT_BATCH) .with_arg(arg) .expire_after(timeout) .call_and_wait(waiter_with_timeout(timeout)) - .await; - x?; + .await?; Ok(()) } @@ -300,38 +296,12 @@ pub async fn post_install_store_assets( let canister_id = info.get_canister_id().expect("Could not find canister ID."); let batch_id = create_batch(agent, &canister_id, timeout).await?; - println!("created batch {}", batch_id); let chunked_assets = make_chunked_assets(agent, &canister_id, timeout, batch_id, asset_locations).await?; - println!("created all chunks"); - println!("committing batch"); commit_batch(agent, &canister_id, timeout, batch_id, chunked_assets).await?; - println!("committed"); - - // let walker = WalkDir::new(output_assets_path).into_iter(); - // for entry in walker { - // let entry = entry?; - // if entry.file_type().is_file() { - // let source = entry.path(); - // let relative: &Path = source - // .strip_prefix(output_assets_path) - // .expect("cannot strip prefix"); - // let content = &std::fs::read(&source)?; - // let path = relative.to_string_lossy().to_string(); - // let blob = candid::Encode!(&path, &content)?; - // - // let method_name = String::from("store"); - // - // agent - // .update(&canister_id, &method_name) - // .with_arg(&blob) - // .expire_after(timeout) - // .call_and_wait(waiter_with_timeout(timeout)) - // .await?; - // } - // } + Ok(()) } @@ -348,6 +318,5 @@ async fn create_batch( .call_and_wait(waiter_with_timeout(timeout)) .await?; let create_batch_response = candid::Decode!(&response, CreateBatchResponse)?; - let batch_id = create_batch_response.batch_id; - Ok(batch_id) + Ok(create_batch_response.batch_id) } diff --git a/src/distributed/assetstorage/Asset.mo b/src/distributed/assetstorage/Asset.mo index f32d91e1a3..f6c0a89ede 100644 --- a/src/distributed/assetstorage/Asset.mo +++ b/src/distributed/assetstorage/Asset.mo @@ -19,6 +19,7 @@ module { public let contentType = initContentType; let encodings = initEncodings; + // Naive encoding selection: of the accepted encodings, pick the first available. public func chooseEncoding(acceptEncodings : [Text]) : ?AssetEncoding { for (acceptEncoding in acceptEncodings.vals()) { switch (encodings.get(acceptEncoding)) { @@ -41,11 +42,9 @@ module { encodings.delete(encodingType) }; - public func toStableAsset() : StableAsset { - { - contentType = contentType; - encodings = Iter.toArray(encodings.entries()); - } + public func toStableAsset() : StableAsset = { + contentType = contentType; + encodings = Iter.toArray(encodings.entries()); }; }; @@ -65,4 +64,4 @@ module { ); (k, a) }; -} \ No newline at end of file +} diff --git a/src/distributed/assetstorage/Batch.mo b/src/distributed/assetstorage/Batch.mo index e0871a0bce..ade6f42f04 100644 --- a/src/distributed/assetstorage/Batch.mo +++ b/src/distributed/assetstorage/Batch.mo @@ -1,8 +1,8 @@ import Debug "mo:base/Debug"; +import HashMap "mo:base/HashMap"; import Int "mo:base/Int"; import Time "mo:base/Time"; -import H "mo:base/HashMap"; import T "Types"; import U "Utils"; @@ -34,7 +34,7 @@ public class Batch(initBatchId: T.BatchId) { // We delete expired batches so that they don't consume space forever after an interrupted install. public class Batches() { var nextBatchId = 1; - let batches = H.HashMap(7, Int.equal, Int.hash); + let batches = HashMap.HashMap(7, Int.equal, Int.hash); public func get(batchId: T.BatchId) : ?Batch { batches.get(batchId) @@ -44,7 +44,7 @@ public class Batches() { batches.delete(batchId) }; - public func startBatch(): Batch { + public func create(): Batch { let batchId = nextBatchId; nextBatchId += 1; let batch = Batch(batchId); @@ -63,4 +63,4 @@ public class Batches() { } } -} \ No newline at end of file +} diff --git a/src/distributed/assetstorage/Chunk.mo b/src/distributed/assetstorage/Chunk.mo index de563ddc6c..ce72f40a52 100644 --- a/src/distributed/assetstorage/Chunk.mo +++ b/src/distributed/assetstorage/Chunk.mo @@ -56,4 +56,4 @@ public class Chunks() { }; } -} \ No newline at end of file +} diff --git a/src/distributed/assetstorage/Main.mo b/src/distributed/assetstorage/Main.mo index 987da30c05..aa3f8d9b9f 100644 --- a/src/distributed/assetstorage/Main.mo +++ b/src/distributed/assetstorage/Main.mo @@ -9,6 +9,7 @@ import Nat8 "mo:base/Nat8"; import Result "mo:base/Result"; import Text "mo:base/Text"; import Time "mo:base/Time"; + import A "Asset"; import B "Batch"; import C "Chunk"; @@ -68,7 +69,7 @@ shared ({caller = creator}) actor class () { throw Error.reject("not authorized"); }; - let batch = batches.startBatch(); + let batch = batches.create(); let chunkId = chunks.create(batch, contents); let create_asset_args : T.CreateAssetArguments = { @@ -101,9 +102,10 @@ shared ({caller = creator}) actor class () { Array.find(authorized, eq) != null }; - // Choose a content encoding from among the accepted encodings and return - // its content, or the first chunk of its content. - // If content.size() > total_length, call get_chunk() get the rest of the content. + // 1. Choose a content encoding from among the accepted encodings. + // 2. Return its content, or the first chunk of its content. + // + // If content.size() > total_length, caller must call get_chunk() get the rest of the content. // All chunks except the last will have the same size as the first chunk. public query func get(arg:{ key: T.Key; @@ -162,13 +164,11 @@ shared ({caller = creator}) actor class () { if (isSafe(caller) == false) throw Error.reject("not authorized"); - Debug.print("create_batch"); - batches.deleteExpired(); chunks.deleteExpired(); { - batch_id = batches.startBatch().batchId; + batch_id = batches.create().batchId; } }; @@ -178,7 +178,7 @@ shared ({caller = creator}) actor class () { } ) : async ({ chunk_id: T.ChunkId }) { - Debug.print("create_chunk(batch " # Int.toText(arg.batch_id) # ", " # Int.toText(arg.content.size()) # " bytes)"); + //Debug.print("create_chunk(batch " # Int.toText(arg.batch_id) # ", " # Int.toText(arg.content.size()) # " bytes)"); if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -193,7 +193,7 @@ shared ({caller = creator}) actor class () { }; public shared ({ caller }) func commit_batch(args: T.CommitBatchArguments) : async () { - Debug.print("commit_batch (" # Int.toText(args.operations.size()) # ")"); + //Debug.print("commit_batch (" # Int.toText(args.operations.size()) # ")"); if (isSafe(caller) == false) throw Error.reject("not authorized"); @@ -224,7 +224,7 @@ shared ({caller = creator}) actor class () { }; func createAsset(arg: T.CreateAssetArguments) : Result.Result<(), Text> { - Debug.print("createAsset(" # arg.key # ")"); + //Debug.print("createAsset(" # arg.key # ")"); switch (assets.get(arg.key)) { case null { let asset = A.Asset( @@ -255,9 +255,9 @@ shared ({caller = creator}) actor class () { if (chunks.size() > 2) { let expectedLength = chunks[0].size(); for (i in Iter.range(1, chunks.size()-2)) { - Debug.print("chunk at index " # Int.toText(i) # " has length " # Int.toText(chunks[i].size()) # " and expected is " # Int.toText(expectedLength) ); + //Debug.print("chunk at index " # Int.toText(i) # " has length " # Int.toText(chunks[i].size()) # " and expected is " # Int.toText(expectedLength) ); if (chunks[i].size() != expectedLength) { - Debug.print("chunk at index " # Int.toText(i) # " with length " # Int.toText(chunks[i].size()) # " does not match expected length " # Int.toText(expectedLength) ); + //Debug.print("chunk at index " # Int.toText(i) # " with length " # Int.toText(chunks[i].size()) # " does not match expected length " # Int.toText(expectedLength) ); return false; } @@ -267,7 +267,7 @@ shared ({caller = creator}) actor class () { }; func setAssetContent(arg: T.SetAssetContentArguments) : Result.Result<(), Text> { - Debug.print("setAssetContent(" # arg.key # ")"); + //Debug.print("setAssetContent(" # arg.key # ")"); switch (assets.get(arg.key)) { case null #err("asset not found"); case (?asset) { @@ -303,7 +303,7 @@ shared ({caller = creator}) actor class () { }; func unsetAssetContent(args: T.UnsetAssetContentArguments) : Result.Result<(), Text> { - Debug.print("unsetAssetContent(" # args.key # ")"); + //Debug.print("unsetAssetContent(" # args.key # ")"); switch (assets.get(args.key)) { case null #err("asset not found"); case (?asset) { @@ -324,7 +324,7 @@ shared ({caller = creator}) actor class () { }; func deleteAsset(args: T.DeleteAssetArguments) : Result.Result<(), Text> { - Debug.print("deleteAsset(" # args.key # ")"); + //Debug.print("deleteAsset(" # args.key # ")"); if (assets.size() > 0) { // avoid div/0 bug https://github.com/dfinity/motoko-base/issues/228 assets.delete(args.key); }; @@ -350,8 +350,4 @@ shared ({caller = creator}) actor class () { #ok(()) }; - - public func version_15() : async() { - }; - }; diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo index e377344c1d..445c98de44 100644 --- a/src/distributed/assetstorage/Types.mo +++ b/src/distributed/assetstorage/Types.mo @@ -44,4 +44,4 @@ module Types { batch_id: BatchId; operations: [BatchOperationKind]; }; -}; \ No newline at end of file +}; diff --git a/src/distributed/assetstorage/Utils.mo b/src/distributed/assetstorage/Utils.mo index 89f681b020..c1b6ee8e98 100644 --- a/src/distributed/assetstorage/Utils.mo +++ b/src/distributed/assetstorage/Utils.mo @@ -1,4 +1,3 @@ -import Debug "mo:base/Debug"; import Hash "mo:base/Hash"; import HashMap "mo:base/HashMap"; import Int "mo:base/Int"; @@ -28,7 +27,6 @@ module Utils { } ); for ((k,_) in entriesToDelete.entries()) { - Debug.print("delete expired"); h.delete(k); }; } From 3eaaf126509dae102486e5786a4f3d495842d95f Mon Sep 17 00:00:00 2001 From: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> Date: Mon, 8 Mar 2021 17:37:22 -0800 Subject: [PATCH 40/48] Update src/dfx/src/lib/installers/assets.rs Use serde_bytes for [u8] Co-authored-by: Yan Chen <48968912+chenyan-dfinity@users.noreply.github.com> --- src/dfx/src/lib/installers/assets.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 0958174068..b6a4b72a4d 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -28,6 +28,7 @@ struct CreateBatchResponse { #[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] struct CreateChunkRequest<'a> { batch_id: u128, + #[serde(with = "serde_bytes")] content: &'a [u8], } From fb14c0945bccbb352c5e5dedc3691c2505014c9f Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 8 Mar 2021 18:51:41 -0800 Subject: [PATCH 41/48] Use candid::Nat rather than u128 --- src/dfx/src/lib/installers/assets.rs | 77 +++++++++++++--------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index b6a4b72a4d..6d2b32477c 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -2,12 +2,12 @@ use crate::lib::canister_info::assets::AssetsCanisterInfo; use crate::lib::canister_info::CanisterInfo; use crate::lib::error::{DfxError, DfxResult}; use crate::lib::waiter::waiter_with_timeout; -use candid::{CandidType, Decode, Encode}; +use candid::{CandidType, Decode, Encode, Nat}; use delay::{Delay, Waiter}; use ic_agent::Agent; use ic_types::Principal; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use std::path::PathBuf; use std::time::Duration; use walkdir::WalkDir; @@ -17,78 +17,78 @@ const CREATE_CHUNK: &str = "create_chunk"; const COMMIT_BATCH: &str = "commit_batch"; const MAX_CHUNK_SIZE: usize = 1_900_000; -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct CreateBatchRequest {} -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug, Deserialize)] struct CreateBatchResponse { - batch_id: u128, + batch_id: Nat, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug, Deserialize)] struct CreateChunkRequest<'a> { - batch_id: u128, + batch_id: Nat, #[serde(with = "serde_bytes")] content: &'a [u8], } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug, Deserialize)] struct CreateChunkResponse { - chunk_id: u128, + chunk_id: Nat, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct GetRequest { key: String, accept_encodings: Vec, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct GetResponse { contents: Vec, content_type: String, content_encoding: String, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct CreateAssetArguments { key: String, content_type: String, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct SetAssetContentArguments { key: String, content_encoding: String, - chunk_ids: Vec, + chunk_ids: Vec, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct UnsetAssetContentArguments { key: String, content_encoding: String, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct DeleteAssetArguments { key: String, } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] +#[derive(CandidType, Debug)] struct ClearArguments {} -#[derive(CandidType, Clone, Debug, Serialize, Deserialize)] +#[derive(CandidType, Debug)] enum BatchOperationKind { CreateAsset(CreateAssetArguments), SetAssetContent(SetAssetContentArguments), - UnsetAssetContent(UnsetAssetContentArguments), + _UnsetAssetContent(UnsetAssetContentArguments), DeleteAsset(DeleteAssetArguments), - Clear(ClearArguments), + _Clear(ClearArguments), } -#[derive(CandidType, Clone, Debug, Default, Serialize, Deserialize)] -struct CommitBatchArguments { - batch_id: u128, +#[derive(CandidType, Debug)] +struct CommitBatchArguments<'a> { + batch_id: &'a Nat, operations: Vec, } @@ -100,16 +100,17 @@ struct AssetLocation { struct ChunkedAsset { asset_location: AssetLocation, - chunk_ids: Vec, + chunk_ids: Vec, } async fn create_chunk( agent: &Agent, canister_id: &Principal, timeout: Duration, - batch_id: u128, + batch_id: &Nat, content: &[u8], -) -> DfxResult { +) -> DfxResult { + let batch_id = batch_id.clone(); let args = CreateChunkRequest { batch_id, content }; let args = candid::Encode!(&args)?; @@ -147,7 +148,7 @@ async fn make_chunked_asset( agent: &Agent, canister_id: &Principal, timeout: Duration, - batch_id: u128, + batch_id: &Nat, asset_location: AssetLocation, ) -> DfxResult { let content = &std::fs::read(&asset_location.source)?; @@ -186,16 +187,14 @@ async fn make_chunked_asset( // }) // works (sometimes) - let mut chunk_ids: Vec = vec![]; + let mut chunk_ids: Vec = vec![]; let chunks = content.chunks(MAX_CHUNK_SIZE); let (num_chunks, _) = chunks.size_hint(); - let mut i: usize = 0; - for data_chunk in chunks { - i += 1; + for (i, data_chunk) in chunks.enumerate() { println!( " {} {}/{} ({} bytes)", &asset_location.relative.to_string_lossy(), - i, + i+1, num_chunks, data_chunk.len() ); @@ -211,7 +210,7 @@ async fn make_chunked_assets( agent: &Agent, canister_id: &Principal, timeout: Duration, - batch_id: u128, + batch_id: &Nat, locs: Vec, ) -> DfxResult> { // this neat futures version works faster in parallel when it works, @@ -232,7 +231,7 @@ async fn commit_batch( agent: &Agent, canister_id: &Principal, timeout: Duration, - batch_id: u128, + batch_id: &Nat, chunked_assets: Vec, ) -> DfxResult { let operations: Vec<_> = chunked_assets @@ -299,18 +298,14 @@ pub async fn post_install_store_assets( let batch_id = create_batch(agent, &canister_id, timeout).await?; let chunked_assets = - make_chunked_assets(agent, &canister_id, timeout, batch_id, asset_locations).await?; + make_chunked_assets(agent, &canister_id, timeout, &batch_id, asset_locations).await?; - commit_batch(agent, &canister_id, timeout, batch_id, chunked_assets).await?; + commit_batch(agent, &canister_id, timeout, &batch_id, chunked_assets).await?; Ok(()) } -async fn create_batch( - agent: &Agent, - canister_id: &Principal, - timeout: Duration, -) -> DfxResult { +async fn create_batch(agent: &Agent, canister_id: &Principal, timeout: Duration) -> DfxResult { let create_batch_args = CreateBatchRequest {}; let response = agent .update(&canister_id, CREATE_BATCH) From a432d2e0dc3573a64baff94aa033f9bdab39d2bf Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 8 Mar 2021 20:00:32 -0800 Subject: [PATCH 42/48] cargo fmt --- src/dfx/src/lib/installers/assets.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 6d2b32477c..3bb00f084a 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -194,7 +194,7 @@ async fn make_chunked_asset( println!( " {} {}/{} ({} bytes)", &asset_location.relative.to_string_lossy(), - i+1, + i + 1, num_chunks, data_chunk.len() ); From 377abbe7a0b88f122e09872e6fab96e27a8c1aea Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Tue, 9 Mar 2021 15:59:53 -0800 Subject: [PATCH 43/48] Add http_request method --- src/distributed/assetstorage/Main.mo | 34 +++++++++++++++++++++++++++ src/distributed/assetstorage/Types.mo | 14 +++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/distributed/assetstorage/Main.mo b/src/distributed/assetstorage/Main.mo index aa3f8d9b9f..c8c54787a7 100644 --- a/src/distributed/assetstorage/Main.mo +++ b/src/distributed/assetstorage/Main.mo @@ -350,4 +350,38 @@ shared ({caller = creator}) actor class () { #ok(()) }; + + public query func http_request(request: T.HttpRequest): async T.HttpResponse { + let path = getPath(request.url); + + let assetEncoding: ?A.AssetEncoding = switch (getAssetEncoding(path)) { + case null getAssetEncoding("index.js"); + case (?found) ?found; + }; + + switch (assetEncoding) { + case null {{ status_code = 404; headers = []; body = "" }}; + case (?c) { + if (c.content.size() > 1) + throw Error.reject("asset too large"); + + { status_code = 200; headers = []; body = c.content[0] } + } + } + }; + + private func getPath(uri: Text): Text { + let splitted = Text.split(uri, #char '?'); + let array = Iter.toArray(splitted); + let path = Text.trimStart(array[0], #char '/'); + path + }; + + private func getAssetEncoding(path: Text): ?A.AssetEncoding { + switch (assets.get(path)) { + case null null; + case (?asset) asset.getEncoding("identity"); + } + }; + }; diff --git a/src/distributed/assetstorage/Types.mo b/src/distributed/assetstorage/Types.mo index 445c98de44..8228f3a462 100644 --- a/src/distributed/assetstorage/Types.mo +++ b/src/distributed/assetstorage/Types.mo @@ -44,4 +44,18 @@ module Types { batch_id: BatchId; operations: [BatchOperationKind]; }; + + public type HeaderField = (Text, Text); + + public type HttpRequest = { + method: Text; + url: Text; + headers: [HeaderField]; + body: Blob; + }; + public type HttpResponse = { + status_code: Nat16; + headers: [HeaderField]; + body: Blob; + }; }; From ea26fc05253ad4803f9ee14372991ce521480d4c Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 10 Mar 2021 16:43:11 -0800 Subject: [PATCH 44/48] Add keys() method for upgrade path; add to changelog --- CHANGELOG.adoc | 16 ++++++++++++++++ e2e/tests-dfx/assetscanister.bash | 19 +++++++++++++++++++ src/distributed/assetstorage/Main.mo | 15 ++++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 9cdedfde07..2295c609e4 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -1,3 +1,19 @@ += 0.6.26 + +== Asset Canister + +- The asset canister can now store assets that exceed the message ingress limit (2 MB) +- The existing interface is left in place for backwards-compatibility, but deprecated. +- These methods are now deprecated: + - retrieve(): use get() and get_chunk() instead + - store(): use call create_batch(), create_chunk(), and commit_batch() instead + - list(): use keys() instead +- A future update will: + - delete the retrieve() method, because it does not support large assets + - change the method signature of store() + - change the method signature of list() + - deprecate keys() + = 0.6.25 == DFX diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index ff9003193b..f837285b69 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -68,3 +68,22 @@ teardown() { assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="large-asset.bin";content_encoding="identity";index=13})' assert_command_fail dfx canister call --query e2e_project_assets get_chunk '(record{key="large-asset.bin";content_encoding="identity";index=14})' } + +@test "list() and keys() return asset keys" { + install_asset assetscanister + + dfx_start + dfx canister create --all + dfx build + dfx canister install e2e_project_assets + + assert_command dfx canister call --query e2e_project_assets list + assert_match 'binary/noise.txt' + assert_match 'text-with-newlines.txt' + assert_match 'sample-asset.txt' + + assert_command dfx canister call --query e2e_project_assets keys + assert_match 'binary/noise.txt' + assert_match 'text-with-newlines.txt' + assert_match 'sample-asset.txt' +} diff --git a/src/distributed/assetstorage/Main.mo b/src/distributed/assetstorage/Main.mo index c8c54787a7..dbde315e89 100644 --- a/src/distributed/assetstorage/Main.mo +++ b/src/distributed/assetstorage/Main.mo @@ -92,11 +92,24 @@ shared ({caller = creator}) actor class () { }; }; - public query func list() : async [T.Path] { + func listKeys(): [T.Path] { let iter = Iter.map<(Text, A.Asset), T.Path>(assets.entries(), func (key, _) = key); Iter.toArray(iter) }; + // deprecated: the signature of this method will change, to take an empty record as + // a parameter and to return an array of records. + // For now, call keys() instead + public query func list() : async [T.Path] { + listKeys() + }; + + // Returns an array of the keys of all assets contained in the asset canister. + // This method will be deprecated after the signature of list() changes. + public query func keys() : async [T.Path] { + listKeys() + }; + func isSafe(caller: Principal) : Bool { func eq(value: Principal): Bool = value == caller; Array.find(authorized, eq) != null From ab502fb618c766626f4b164df9ea78efff0aa645 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Wed, 10 Mar 2021 17:01:22 -0800 Subject: [PATCH 45/48] removed an extra word --- CHANGELOG.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 2295c609e4..8de60ccc39 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -6,7 +6,7 @@ - The existing interface is left in place for backwards-compatibility, but deprecated. - These methods are now deprecated: - retrieve(): use get() and get_chunk() instead - - store(): use call create_batch(), create_chunk(), and commit_batch() instead + - store(): use create_batch(), create_chunk(), and commit_batch() instead - list(): use keys() instead - A future update will: - delete the retrieve() method, because it does not support large assets From d895d0ba1a667819019a3a52678ce88512552dd6 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 11 Mar 2021 11:56:34 -0800 Subject: [PATCH 46/48] ok maybe 25 MB was always going to take too long on CI --- e2e/tests-dfx/assetscanister.bash | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index 168459f59c..fdd19d134c 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -71,19 +71,19 @@ CHERRIES" "$stdout" dfx build dfx canister install e2e_project_assets - dd if=/dev/urandom of=src/e2e_project_assets/assets/large-asset.bin bs=1000000 count=25 + dd if=/dev/urandom of=src/e2e_project_assets/assets/large-asset.bin bs=1000000 count=6 dfx deploy assert_command dfx canister call --query e2e_project_assets get '(record{key="/large-asset.bin";accept_encodings=vec{"identity"}})' - assert_match 'total_length = 25_000_000' + assert_match 'total_length = 6_000_000' assert_match 'content_type = "application/octet-stream"' assert_match 'content_encoding = "identity"' - assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=4})' + assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=2})' - assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=13})' - assert_command_fail dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=14})' + assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=3})' + assert_command_fail dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=4})' } @test "list() and keys() return asset keys" { From bd5a75b7bb70522a702a1c9e3f94a4710ba65172 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 11 Mar 2021 14:00:02 -0800 Subject: [PATCH 47/48] disable large asset test on ic-ref due to wasm interpreter efficiency --- e2e/tests-dfx/assetscanister.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index fdd19d134c..045e0b1de8 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -64,6 +64,8 @@ CHERRIES" "$stdout" } @test 'can store arbitrarily large files' { + [ "$USE_IC_REF" ] && skip "skip for ic-ref" # this takes too long for ic-ref's wasm interpreter + install_asset assetscanister dfx_start From d4f4ed175dcdd967f13242753fbb9bc3274afbf5 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Thu, 11 Mar 2021 14:00:16 -0800 Subject: [PATCH 48/48] serde_bytes for GetResponse --- src/dfx/src/lib/installers/assets.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dfx/src/lib/installers/assets.rs b/src/dfx/src/lib/installers/assets.rs index 4560e801a4..46a57ce578 100644 --- a/src/dfx/src/lib/installers/assets.rs +++ b/src/dfx/src/lib/installers/assets.rs @@ -43,8 +43,9 @@ struct GetRequest { accept_encodings: Vec, } -#[derive(CandidType, Debug)] +#[derive(CandidType, Debug, Deserialize)] struct GetResponse { + #[serde(with = "serde_bytes")] contents: Vec, content_type: String, content_encoding: String,