feat: allow upload/retrieve of assets of arbitrary size from asset canister

CHANGELOG.adoc

@@ -19,7 +19,15 @@ This commit also upgrades tokio and reqwest in order to work correctly. There ar
 
 Renamed the `project_name` in our own generated assets to `canister_name`, for things that are generated during canister build (and not project generation).
 
----
+== Asset Canister
+
+=== feat: The asset canister can now store assets that exceed the message ingress limit (2 MB)
+
+* Please note that neither the JS agent nor the HTTP server have been updated yet to server such large assets.
+* The existing interface is left in place for backwards-compatibility, but deprecated:
+** retrieve(): use get() and get_chunk() instead
+** store(): use create_batch(), create_chunk(), and commit_batch() instead
+** list(): use keys() instead
 
 = 0.6.25
 

distributed-canisters.nix

@@ -12,19 +12,19 @@ pkgs.runCommandNoCCLocal "distributed-canisters" {
 } ''
   mkdir -p $out
 
-  for canister_mo in ${distributed}/*.mo; do
-    canister_name=$(basename -s .mo $canister_mo)
+  for canister_dir in $(find ${distributed} -mindepth 1 -maxdepth 1 -type d); do
+    canister_name=$(basename $canister_dir)
 
     build_dir=$out/$canister_name
     mkdir -p $build_dir
 
     $moc/bin/moc \
-       $canister_mo \
+       $canister_dir/Main.mo \
        -o $build_dir/$canister_name.did \
        --idl \
        --package base $base
     $moc/bin/moc \
-       $canister_mo \
+       $canister_dir/Main.mo \
        -o $build_dir/$canister_name.wasm \
        -c --release \
        --package base $base

e2e/tests-dfx/assetscanister.bash

@@ -62,3 +62,47 @@ teardown() {
 it's cherry season
 CHERRIES" "$stdout"
 }
+
+@test 'can store arbitrarily large files' {
+    [ "$USE_IC_REF" ] && skip "skip for ic-ref" # this takes too long for ic-ref's wasm interpreter
+
+    install_asset assetscanister
+
+    dfx_start
+    dfx canister create --all
+    dfx build
+    dfx canister install e2e_project_assets
+
+    dd if=/dev/urandom of=src/e2e_project_assets/assets/large-asset.bin bs=1000000 count=6
+
+    dfx deploy
+
+    assert_command dfx canister call --query e2e_project_assets get '(record{key="/large-asset.bin";accept_encodings=vec{"identity"}})'
+    assert_match 'total_length = 6_000_000'
+    assert_match 'content_type = "application/octet-stream"'
+    assert_match 'content_encoding = "identity"'
+
+    assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=2})'
+
+    assert_command dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=3})'
+    assert_command_fail dfx canister call --query e2e_project_assets get_chunk '(record{key="/large-asset.bin";content_encoding="identity";index=4})'
+}
+
+@test "list() and keys() return asset keys" {
+    install_asset assetscanister
+
+    dfx_start
+    dfx canister create --all
+    dfx build
+    dfx canister install e2e_project_assets
+
+    assert_command dfx canister call --query e2e_project_assets list
+    assert_match '"/binary/noise.txt"'
+    assert_match '"/text-with-newlines.txt"'
+    assert_match '"/sample-asset.txt"'
+
+    assert_command dfx canister call --query e2e_project_assets keys
+    assert_match '"/binary/noise.txt"'
+    assert_match '"/text-with-newlines.txt"'
+    assert_match '"/sample-asset.txt"'
+}