feat (yaml2candid): Support blobs input as hex or base64 (#106)

# Motivation
`idl2json` supports encoding blobs as hex but the reverse is not true:
`yaml2candid` does not accept byte arrays expressed as hex. This is
inconvenient, to put it mildly.

# Changes
- Support blobs expressed as hex or base64.

# Tests
- Unit tests are included with 100% code coverage of the added lines.

Cargo.toml

@@ -14,6 +14,8 @@ resolver = "2"
 version = "0.10.1"
 
 [workspace.dependencies]
+base64 = { version = "0.22.1" }
 candid = { version = "0.10.8" }
 candid_parser = { version = "0.1.0" }
-num-bigint = { version = "0.4.5" }
+hex = { version = "0.4.3" }
+num-bigint = { version = "0.4.5" }

src/idl2json/Cargo.toml

@@ -12,20 +12,19 @@ readme = "../../README.md"
 
 categories = ["encoding", "parsing"]
 keywords = ["internet-computer", "idl", "candid", "dfinity", "json"]
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 candid = { workspace = true }
 candid_parser = { workspace = true }
+clap = { version = "4", features = [ "derive" ], optional = true }
 serde_json = "^1.0"
 sha2 = { version = "0.10.8", optional = true }
-clap = { version = "4", features = [ "derive" ], optional = true }
 
 [dev-dependencies]
 json-patch = "0.2.7"
-serde = "1"
 num-bigint = "0.4.6"
+serde = "1"
 
 [features]
 default = ["crypto"]

src/idl2json_cli/Cargo.toml

@@ -12,21 +12,20 @@ readme = "../../README.md"
 
 categories = ["encoding", "parsing"]
 keywords = ["internet-computer", "idl", "candid", "dfinity", "json"]
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [[bin]]
 name = "idl2json"
 path = "src/main.rs"
 
 [dependencies]
+anyhow = "1"
 candid = { workspace = true }
 candid_parser = { workspace = true }
 clap = { version = "4.5.8", features = [ "derive" ] }
 fn-error-context = "0.2.1"
+idl2json = { path = "../idl2json", version = "0.10.1", features = ["clap", "crypto"] }
 serde_json = "^1.0"
-idl2json = { path = "../idl2json", version="0.10.1", features = ["clap", "crypto"] }
-anyhow = "1"
 
 [build-dependencies]
 anyhow = "1"

src/yaml2candid/Cargo.toml

@@ -2,16 +2,17 @@
 name = "yaml2candid"
 version = { workspace = true }
 edition = "2021"
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1"
+base64 = { workspace = true }
 candid = { workspace = true }
 candid_parser = { workspace = true }
+hex = { workspace = true }
+num-bigint = { workspace = true }
 serde = "1"
 serde_yaml = "0.9"
-anyhow = "1"
-num-bigint = { workspace = true }
 
 [dev-dependencies]
-pretty_assertions = "1.4.0"
+pretty_assertions = "1.4.0"

src/yaml2candid/src/lib.rs

@@ -3,6 +3,7 @@
 #![deny(clippy::panic)]
 #![deny(clippy::unwrap_used)]
 use anyhow::{anyhow, bail, Context};
+use base64::Engine as _;
 use candid::{
     types::value::{IDLField, IDLValue, VariantValue},
     Principal,
@@ -251,8 +252,25 @@ impl Yaml2Candid {
                     .enumerate()
                     .map(|(index, val)| self.convert(typ, val).with_context(|| format!("Failed to convert element #{index}")))
                     .collect::<Result<Vec<_>, _>>()?,
-            ))} else {
-                bail!("Expected a sequence for vec type, got: {data:?}")
+            ))} else if matches!(**typ, IDLType::PrimT(candid_parser::types::PrimType::Nat8)) {
+                // Byte vectors MAY be represented in different ways, such as hex or base64 strings
+                const BASE_64_PREFIX: &str="base64,";
+                const HEX_PREFIX: &str="0x";
+                if let YamlValue::String(value) = data {
+                  if let Some(base64_str) = value.strip_prefix(BASE_64_PREFIX) {
+                    let bytes = base64::engine::general_purpose::STANDARD.decode(base64_str).with_context(|| "Failed to interpret base64 string in JSON/YAML as Candid byte vector")?;
+                    Ok(IDLValue::Vec(bytes.iter().map(|&b| IDLValue::Nat8(b)).collect::<Vec<_>>()))
+                  } else if let Some(hex_str) = value.strip_prefix(HEX_PREFIX) {
+                    let bytes = hex::decode(hex_str).with_context(|| "Failed to interpret hex string in JSON/YAML as Candid byte vector")?;
+                    Ok(IDLValue::Vec(bytes.iter().map(|&b| IDLValue::Nat8(b)).collect::<Vec<_>>()))
+                  } else {
+                    bail!("Unknown encoding for byte vector as string starting: {}  Please prefix string encoded byte vectors with one of '{BASE_64_PREFIX}' and '{HEX_PREFIX}'.", &value[0..6])
+                  }
+                } else {
+                    bail!("Expected vector of bytes encoded as one of: Vec<number>, hex string prefixed with '{HEX_PREFIX}', base64 string prefixed with '{BASE_64_PREFIX}'.");
+                }
+            } else {
+                bail!("Expected a sequence of {typ:?}, got: {data:?}")
             },
             IDLType::VariantT(types) => if let YamlValue::Mapping(value) = data {
                 types

src/yaml2candid/src/test.rs

@@ -1,4 +1,5 @@
 //! Tests converting from YAML to Candid, especially extreme values of primitive types.
+#![allow(clippy::panic)] // Tests are allowed to panic!
 use anyhow::Context;
 use candid::types::{value::IDLField, Label};
 use candid_parser::types::TypeField;
@@ -494,6 +495,38 @@ fn can_convert() {
                 IDLValue::Int8(3),
             ]),
         },
+        TestVec {
+            description: "hex encoded blob",
+            typ: IDLType::VecT(Box::new(IDLType::PrimT(
+                candid_parser::types::PrimType::Nat8,
+            ))),
+            data: YamlValue::from("0x010203090a1000"),
+            expected_result: IDLValue::Vec(vec![
+                IDLValue::Nat8(1),
+                IDLValue::Nat8(2),
+                IDLValue::Nat8(3),
+                IDLValue::Nat8(9),
+                IDLValue::Nat8(10),
+                IDLValue::Nat8(16),
+                IDLValue::Nat8(0),
+            ]),
+        },
+        TestVec {
+            description: "base64 encoded blob",
+            typ: IDLType::VecT(Box::new(IDLType::PrimT(
+                candid_parser::types::PrimType::Nat8,
+            ))),
+            data: YamlValue::from("base64,AQIDCQoQAA=="),
+            expected_result: IDLValue::Vec(vec![
+                IDLValue::Nat8(1),
+                IDLValue::Nat8(2),
+                IDLValue::Nat8(3),
+                IDLValue::Nat8(9),
+                IDLValue::Nat8(10),
+                IDLValue::Nat8(16),
+                IDLValue::Nat8(0),
+            ]),
+        },
         TestVec {
             description: "Some(5) in canonical form",
             typ: IDLType::OptT(Box::new(IDLType::PrimT(
@@ -659,3 +692,49 @@ fn can_convert() {
         assert_named_conversion_is(&converter, &typ, &data, expected_result, description);
     }
 }
+
+/// A conversion that should fail.
+struct ErrorTestVec {
+    typ: IDLType,
+    data: YamlValue,
+    expected_error: &'static str,
+}
+impl ErrorTestVec {
+    pub fn should_fail(&self) {
+        let Self {
+            typ,
+            data,
+            expected_error,
+        } = self;
+        let converter = Yaml2Candid::default();
+        let result = converter.convert(typ, data);
+        if let Err(e) = result {
+            assert!(e.to_string().contains(expected_error))
+        } else {
+            panic!("Converting {data:?} to {typ:?} should fail.");
+        }
+    }
+}
+
+#[test]
+fn unsupported_blob_encoding_should_fail() {
+    ErrorTestVec{
+        typ: IDLType::VecT(Box::new(IDLType::PrimT(
+            candid_parser::types::PrimType::Nat8,
+        ))),
+        data: YamlValue::from("010203090a1000"),
+        expected_error: "Unknown encoding for byte vector as string starting: 010203  Please prefix string encoded byte vectors"
+    }.should_fail();
+}
+
+#[test]
+fn unsupported_blob_type_should_fail() {
+    ErrorTestVec {
+        typ: IDLType::VecT(Box::new(IDLType::PrimT(
+            candid_parser::types::PrimType::Nat8,
+        ))),
+        data: YamlValue::from(9),
+        expected_error: "Expected vector of bytes encoded as one of",
+    }
+    .should_fail();
+}

src/yaml2candid_cli/Cargo.toml

@@ -2,7 +2,6 @@
 name = "yaml2candid_cli"
 version = { workspace = true }
 edition = "2021"
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]