From 4f47a83b5f62619290aa8feb3d1f8351233b7524 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Wed, 24 Apr 2024 19:23:24 +0900 Subject: [PATCH 01/50] adding test utility --- .gitignore | 2 + .vscode/settings.json | 1 + spec/brgen_json_schema.json | 2 +- spec/brgen_test_info_schema.json | 41 +++++ src/tool/brgen/main.go | 14 +- testutil/Cargo.lock | 275 +++++++++++++++++++++++++++++++ testutil/Cargo.toml | 11 ++ testutil/src/main.rs | 11 ++ 8 files changed, 350 insertions(+), 7 deletions(-) create mode 100644 spec/brgen_test_info_schema.json create mode 100644 testutil/Cargo.lock create mode 100644 testutil/Cargo.toml create mode 100644 testutil/src/main.rs diff --git a/.gitignore b/.gitignore index cdcd575d..dbd303f7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ go_dump_env.bat license_cache pkg/ stats.json +zig-cache/ +zig-out/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 5072a237..7c246f9e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -148,6 +148,7 @@ ".\\ast2rust\\Cargo.toml", ".\\src\\tool\\json2rust\\Cargo.toml", ".\\src\\tool\\json2llvm\\Cargo.toml", + ".\\testutil\\Cargo.toml", ], "brgen-lsp.src2json" :"./tool/src2json.exe", //"go.buildTags": "linux", diff --git a/spec/brgen_json_schema.json b/spec/brgen_json_schema.json index a6212948..1ae66173 100644 --- a/spec/brgen_json_schema.json +++ b/spec/brgen_json_schema.json @@ -24,7 +24,7 @@ }, "test_info_output": { "type": "string", - "description": "Path to the test info output file (default: no output)" + "description": "Path to the test info output file (default: no output). schema is defined in brgen_test_info_schema.json" }, "warnings": { "type": "object", diff --git a/spec/brgen_test_info_schema.json b/spec/brgen_test_info_schema.json new file mode 100644 index 00000000..8f3fc6c9 --- /dev/null +++ b/spec/brgen_test_info_schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "description": "brgen(BinaRy encoder/decoder GENerator driver) output information file schema", + "properties": { + "total_count": { + "type": "integer", + "description": "Total count of output files" + }, + "error_count": { + "type": "integer", + "description": "Count of output files with errors" + }, + "time": { + "type": "string", + "description": "Time taken to generate output files" + }, + "generated_files": { + "type": "array", + "description": "List of generated output files", + "items": { + "type": "object", + "properties": { + "dir": { + "type": "string", + "description": "Directory containing the output file" + }, + "base": { + "type": "string", + "description": "Output file name without extension" + }, + "suffix": { + "type": "string", + "description": "Output file extension" + } + } + } + } + }, + "additionalProperties": false +} diff --git a/src/tool/brgen/main.go b/src/tool/brgen/main.go index b0681279..8c1bdfd0 100644 --- a/src/tool/brgen/main.go +++ b/src/tool/brgen/main.go @@ -123,6 +123,13 @@ func init() { var exitCode int +type TestInfo struct { + TotalCount uint32 `json:"total_count"` + ErrorCount uint32 `json:"error_count"` + Time string `json:"time"` + DirAndBase []*DirBaseSuffix `json:"generated_files"` +} + func main() { defer os.Exit(exitCode) log.SetPrefix("brgen: ") @@ -216,12 +223,7 @@ func main() { log.Fatal(err) } defer fp.Close() - var info struct { - TotalCount uint32 `json:"total_count"` - ErrorCount uint32 `json:"error_count"` - Time string `json:"time"` - DirAndBase []*DirBaseSuffix `json:"generated_files"` - } + var info TestInfo info.TotalCount = totalCount.Load() info.ErrorCount = errCount.Load() info.Time = elapsed.String() diff --git a/testutil/Cargo.lock b/testutil/Cargo.lock new file mode 100644 index 00000000..8d73887a --- /dev/null +++ b/testutil/Cargo.lock @@ -0,0 +1,275 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anstream" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "clap" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "proc-macro2" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "serde" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "testutil" +version = "0.1.0" +dependencies = [ + "clap", + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/testutil/Cargo.toml b/testutil/Cargo.toml new file mode 100644 index 00000000..4e71447e --- /dev/null +++ b/testutil/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "testutil" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { version = "4.5.4", features = ["derive"] } +serde = "1.0.196" +serde_json = "1.0.116" \ No newline at end of file diff --git a/testutil/src/main.rs b/testutil/src/main.rs new file mode 100644 index 00000000..b9411d33 --- /dev/null +++ b/testutil/src/main.rs @@ -0,0 +1,11 @@ +use clap::{arg, Parser}; + +#[derive(Parser)] +struct Args { + #[arg(long, short)] + spec: bool, +} + + + +fn main() {} From 1bc05781ba6f8e39817951dbe362c613260bd00c Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Wed, 24 Apr 2024 21:40:56 +0900 Subject: [PATCH 02/50] adding schema --- spec/brgen_genrated_mapping_schema.json | 62 +++++++++++++++++++++++++ testutil/src/main.rs | 17 ++++++- 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 spec/brgen_genrated_mapping_schema.json diff --git a/spec/brgen_genrated_mapping_schema.json b/spec/brgen_genrated_mapping_schema.json new file mode 100644 index 00000000..a688ba82 --- /dev/null +++ b/spec/brgen_genrated_mapping_schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "description": "brgen(BinaRy encoder/decoder GENerator driver) generated file information", + "properties": { + "structs": { + "type": "array", + "description": "List of structs", + "item": { + "type": "string", + "description": "Struct name" + } + }, + "line_map": { + "type": "array", + "description": "Line number mappings", + "item": { + "type": "object", + "description": "Line number mapping", + "item": { + "line": { + "type": "integer", + "description": "Line number of generated file" + }, + "loc": { + "type": "object", + "description": "Location of source file", + "item": { + "file": { + "type": "integer", + "description": "File index" + }, + "line": { + "type": "integer", + "description": "Line number of source file" + }, + "col": { + "type": "integer", + "description": "Column number of source file" + }, + "pos": { + "type": "object", + "description": "Position of source file", + "item": { + "begin": { + "type": "integer", + "description": "Begin position of source file" + }, + "end": { + "type": "integer", + "description": "End position of source file" + } + } + } + } + } + } + } + } + }, + "additionalProperties": false +} diff --git a/testutil/src/main.rs b/testutil/src/main.rs index b9411d33..224d0bcd 100644 --- a/testutil/src/main.rs +++ b/testutil/src/main.rs @@ -1,11 +1,24 @@ use clap::{arg, Parser}; +use serde::{Deserialize, Serialize}; #[derive(Parser)] struct Args { - #[arg(long, short)] - spec: bool, + #[arg(long, short('f'))] + test_info_file: String, } +#[derive(Serialize, Deserialize)] +struct GeneratedFile { + dir: String, + base: String, + suffix: String, +} +struct TestInfo { + total_count: u64, + err_count: u64, + time: String, + generated_files: GeneratedFile, +} fn main() {} From bfc8e010679fa8a116feb6166f878e7cc9762269 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Thu, 25 Apr 2024 02:44:04 +0900 Subject: [PATCH 03/50] adding testutil --- testutil/Cargo.lock | 121 ++++++++++++++++++++++++++ testutil/Cargo.toml | 7 +- testutil/src/main.rs | 181 ++++++++++++++++++++++++++++++++++++--- testutil/src/testutil.rs | 96 +++++++++++++++++++++ 4 files changed, 391 insertions(+), 14 deletions(-) create mode 100644 testutil/src/testutil.rs diff --git a/testutil/Cargo.lock b/testutil/Cargo.lock index 8d73887a..4233584b 100644 --- a/testutil/Cargo.lock +++ b/testutil/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.6.13" @@ -50,6 +59,21 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ast2rust" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clap" version = "4.5.4" @@ -96,6 +120,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "getrandom" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "heck" version = "0.5.0" @@ -108,6 +143,24 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.81" @@ -126,6 +179,65 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + [[package]] name = "ryu" version = "1.0.17" @@ -184,7 +296,10 @@ dependencies = [ name = "testutil" version = "0.1.0" dependencies = [ + "ast2rust", "clap", + "rand", + "regex", "serde", "serde_json", ] @@ -201,6 +316,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/testutil/Cargo.toml b/testutil/Cargo.toml index 4e71447e..fee73625 100644 --- a/testutil/Cargo.toml +++ b/testutil/Cargo.toml @@ -7,5 +7,8 @@ edition = "2021" [dependencies] clap = { version = "4.5.4", features = ["derive"] } -serde = "1.0.196" -serde_json = "1.0.116" \ No newline at end of file +serde = { version="1.0.196" ,features = ["derive"]} +serde_json = "1.0.116" +ast2rust = {path ="../ast2rust" } +regex = "1.10.4" +rand = "0.8.5" diff --git a/testutil/src/main.rs b/testutil/src/main.rs index 224d0bcd..f6501463 100644 --- a/testutil/src/main.rs +++ b/testutil/src/main.rs @@ -1,24 +1,181 @@ +use std::{ + collections::HashMap, + env, + ffi::OsStr, + fs, os, + path::{Path, PathBuf}, +}; + use clap::{arg, Parser}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use testutil::TestSchedule; +mod testutil; +use rand::{ + self, + distributions::{Alphanumeric, DistString}, +}; #[derive(Parser)] struct Args { #[arg(long, short('f'))] test_info_file: String, + #[arg(long, short('c'))] + test_config_file: String, } +type Error = Box; -#[derive(Serialize, Deserialize)] -struct GeneratedFile { - dir: String, - base: String, - suffix: String, +struct TestScheduler { + template_files: HashMap, + tmpdir: Option, } -struct TestInfo { - total_count: u64, - err_count: u64, - time: String, - generated_files: GeneratedFile, +impl TestScheduler { + fn read_template(&mut self, path: &str) -> Result { + if let Some(x) = self.template_files.get(path) { + return Ok(x.clone()); + } else { + let t = fs::read_to_string(path)?; + self.template_files.insert(path.to_string(), t); + Ok(self.template_files.get(path).unwrap().clone()) + } + } + + fn get_tmp_dir<'a>(&'a mut self) -> PathBuf { + if let Some(x) = self.tmpdir.as_ref() { + x.clone() + } else { + let dir = env::temp_dir(); + let mut rng = rand::thread_rng(); + let random_str = Alphanumeric.sample_string(&mut rng, 32); + let dir = dir.join(random_str); + self.tmpdir = Some(dir); + self.tmpdir.as_ref().unwrap().clone() + } + } + + fn prepare_content<'a>(&mut self, sched: &TestSchedule<'a>) -> Result { + // get template and replace with target + let template = self.read_template(&sched.runner.test_template)?; + let replace_with = &sched.runner.replace_struct_name; + let template = template.replace(replace_with, &sched.input.format_name); + let replace_with = &sched.runner.replace_file_name; + let path = format!("{}/{}", sched.file.dir, sched.file.base); + let instance = template.replace(replace_with, &path); + Ok(instance) + } + + fn create_input_file<'a>( + &mut self, + sched: &TestSchedule<'a>, + instance: String, + ) -> Result<(PathBuf, PathBuf), Error> { + let tmp_dir = self.get_tmp_dir(); + let tmp_dir = tmp_dir.join(&sched.file.base); + let tmp_dir = tmp_dir.join(&sched.input.format_name); + let tmp_dir = tmp_dir.join(&sched.file.suffix); + fs::create_dir_all(&tmp_dir)?; + let input_file = tmp_dir.join(&sched.runner.build_input_name); + let output_file = tmp_dir.join(&sched.runner.build_output_name); + fs::write(input_file, instance)?; + Ok((tmp_dir, output_file)) + } + + pub fn run_test_schedule<'a>(&mut self, sched: &TestSchedule<'a>) -> Result<(), Error> { + let instance = self.prepare_content(sched)?; + + let (tmp_dir, output) = self.create_input_file(sched, instance)?; + + Ok(()) + } } -fn main() {} +fn main() -> Result<(), Error> { + let parsed = Args::parse(); + let test_config = fs::read_to_string(Path::new(&parsed.test_config_file))?; + let mut d1 = serde_json::Deserializer::from_str(&test_config); + let test_config = testutil::TestConfig::deserialize(&mut d1)?; + let mut ext_set = std::collections::HashMap::new(); + let mut file_format_list = std::collections::HashMap::new(); + for runner in &test_config.runners { + let mut has_input = false; + let mut has_output = false; + let mut has_exec = false; + for cmd in &runner.build_command { + if cmd == "$INPUT" { + has_input = true; + } + if cmd == "$OUTPUT" { + has_output = true; + } + } + if !has_input || !has_output { + eprint!( + "build command not contains $INPUT or $OUTPUT; should contain both {:?}", + runner.build_command + ); + continue; + } + has_input = false; + has_output = false; + has_exec = false; + for cmd in &runner.run_command { + if cmd == "$INPUT" { + has_input = true; + } + if cmd == "$OUTPUT" { + has_output = true; + } + if cmd == "$EXEC" { + has_exec = true; + } + } + if !has_input || !has_output || !has_exec { + eprint!( + "run command not contains $INPUT,$OUTPUT or $EXEC; should contain all {:?}", + runner.run_command + ); + continue; + } + ext_set.insert(runner.suffix.clone(), runner.clone()); + } + for input in &test_config.inputs { + file_format_list.insert( + (input.file_base.clone(), input.format_name.clone()), + input.clone(), + ); + } + let mut sched = Vec::new(); + let test_info = fs::read_to_string(Path::new(&parsed.test_info_file))?; + let mut d2 = serde_json::Deserializer::from_str(&test_info); + let test_info = testutil::TestInfo::deserialize(&mut d2)?; + for file in &test_info.generated_files { + if file.suffix != ".json" { + continue; + } + let ext = file.base.split('.').last().unwrap(); + let runner = match ext_set.get(ext) { + Some(x) => x, + None => continue, + }; + let path = file.into_path(); + let content = fs::read_to_string(&path)?; + let mut d = serde_json::Deserializer::from_str(&content); + let data = match testutil::GeneratedData::deserialize(&mut d) { + Ok(file) => file, + Err(e) => { + eprintln!("failed to deserialize {}: {}", path, e); + continue; + } + }; + for s in &data.structs { + if let Some(input) = file_format_list.get(&(file.base.clone(), s.clone())) { + sched.push(TestSchedule { + runner: runner, + input: input, + file: file, + }) + } + } + } + Ok(()) +} diff --git a/testutil/src/testutil.rs b/testutil/src/testutil.rs new file mode 100644 index 00000000..cc5ffce8 --- /dev/null +++ b/testutil/src/testutil.rs @@ -0,0 +1,96 @@ +use std::path::{Path, PathBuf}; + +use ast2rust::ast; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] + +pub struct LineMap { + pub line: u64, + + pub loc: ast::Loc, +} + +#[derive(Serialize, Deserialize)] + +pub struct GeneratedData { + pub structs: Vec, + + pub line_map: Vec, +} + +#[derive(Serialize, Deserialize)] + +pub struct GeneratedFileInfo { + pub dir: String, + + pub base: String, + + pub suffix: String, +} + +#[derive(Serialize, Deserialize)] +pub struct TestInfo { + pub total_count: u64, + + pub err_count: u64, + + pub time: String, + + pub generated_files: Vec, +} + +impl GeneratedFileInfo { + pub fn into_path(&self) -> String { + format!("{}/{}.{}", self.dir, self.base, self.suffix) + } +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct TestRunner { + // file suffix of generated files + pub suffix: String, + // test template file + pub test_template: String, + // replace target of struct in test_template file + pub replace_struct_name: String, + // replace target of file name of test target file + pub replace_file_name: String, + + pub build_input_name: String, + pub build_output_name: String, + // command to build test + // $INPUT is replaced with test file path + // $OUTPUT is replaced with output file path + pub build_command: Vec, + // command to run test + // $INPUT is replaced with test input file path + // $EXEC is replaced with test exec file path that is built by build_command + // $OUTPUT is replaced with test output file path + pub run_command: Vec, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct TestInput { + // input binary file + pub binary: String, + // test target format name + pub format_name: String, + // file base name that contains format_name format + pub file_base: String, + // this input is failure case + pub failure_case: bool, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct TestConfig { + pub runners: Vec, + // test input binary file + pub inputs: Vec, +} + +pub struct TestSchedule<'a> { + pub input: &'a TestInput, + pub runner: &'a TestRunner, + pub file: &'a GeneratedFileInfo, +} From 764310defb1a6c0f23b34b931728ac04282f82d5 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Thu, 25 Apr 2024 03:02:45 +0900 Subject: [PATCH 04/50] adding --- testutil/src/main.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/testutil/src/main.rs b/testutil/src/main.rs index f6501463..86e1fb22 100644 --- a/testutil/src/main.rs +++ b/testutil/src/main.rs @@ -4,6 +4,7 @@ use std::{ ffi::OsStr, fs, os, path::{Path, PathBuf}, + process, }; use clap::{arg, Parser}; @@ -68,7 +69,7 @@ impl TestScheduler { &mut self, sched: &TestSchedule<'a>, instance: String, - ) -> Result<(PathBuf, PathBuf), Error> { + ) -> Result<(PathBuf, PathBuf, PathBuf), Error> { let tmp_dir = self.get_tmp_dir(); let tmp_dir = tmp_dir.join(&sched.file.base); let tmp_dir = tmp_dir.join(&sched.input.format_name); @@ -76,14 +77,28 @@ impl TestScheduler { fs::create_dir_all(&tmp_dir)?; let input_file = tmp_dir.join(&sched.runner.build_input_name); let output_file = tmp_dir.join(&sched.runner.build_output_name); - fs::write(input_file, instance)?; - Ok((tmp_dir, output_file)) + fs::write(&input_file, instance)?; + Ok((tmp_dir, input_file, output_file)) + } + + fn exec_build<'a>(&mut self, sched: &TestSchedule<'a>, input: &PathBuf, output: &PathBuf) { + let mut cmd = sched.runner.build_command.clone(); + for c in &mut cmd { + if c == "$INPUT" { + *c = input.to_str().unwrap().to_string(); + } + if c == "$OUTPUT" { + *c = output.to_str().unwrap().to_string(); + } + } + let mut r = process::Command::new(&cmd[0]); + r.args(&cmd[1..]); } pub fn run_test_schedule<'a>(&mut self, sched: &TestSchedule<'a>) -> Result<(), Error> { let instance = self.prepare_content(sched)?; - let (tmp_dir, output) = self.create_input_file(sched, instance)?; + let (tmp_dir, input, output) = self.create_input_file(sched, instance)?; Ok(()) } From f8b8c6d862537df4f3baa6020b7aca18bca508e2 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:22:02 +0900 Subject: [PATCH 05/50] add test util --- testutil/src/main.rs | 150 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 3 deletions(-) diff --git a/testutil/src/main.rs b/testutil/src/main.rs index 86e1fb22..59746ec6 100644 --- a/testutil/src/main.rs +++ b/testutil/src/main.rs @@ -28,9 +28,18 @@ type Error = Box; struct TestScheduler { template_files: HashMap, tmpdir: Option, + input_binaries: HashMap>, } impl TestScheduler { + fn new() -> Self { + Self { + template_files: HashMap::new(), + tmpdir: None, + input_binaries: HashMap::new(), + } + } + fn read_template(&mut self, path: &str) -> Result { if let Some(x) = self.template_files.get(path) { return Ok(x.clone()); @@ -41,6 +50,16 @@ impl TestScheduler { } } + fn read_input_binary(&mut self, path: &str) -> Result, Error> { + if let Some(x) = self.input_binaries.get(path) { + return Ok(x.clone()); + } else { + let t = fs::read_to_string(path)?; + self.input_binaries.insert(path.to_string(), t); + Ok(self.input_binaries.get(path).unwrap().clone()) + } + } + fn get_tmp_dir<'a>(&'a mut self) -> PathBuf { if let Some(x) = self.tmpdir.as_ref() { x.clone() @@ -81,18 +100,62 @@ impl TestScheduler { Ok((tmp_dir, input_file, output_file)) } - fn exec_build<'a>(&mut self, sched: &TestSchedule<'a>, input: &PathBuf, output: &PathBuf) { - let mut cmd = sched.runner.build_command.clone(); - for c in &mut cmd { + fn replace_cmd( + cmd: &mut Vec, + tmp_dir: &PathBuf, + input: &PathBuf, + output: &PathBuf, + exec: Option<&PathBuf>, + ) { + for c in cmd { if c == "$INPUT" { *c = input.to_str().unwrap().to_string(); } if c == "$OUTPUT" { *c = output.to_str().unwrap().to_string(); } + if c == "$EXEC" { + if let Some(e) = exec { + *c = e.to_str().unwrap().to_string(); + } + } + if c == "$TMPDIR" { + *c = tmp_dir.to_str().unwrap().to_string(); + } } + } + + fn exec_cmd<'a>( + &mut self, + base: &Vec, + tmp_dir: &PathBuf, + input: &PathBuf, + output: &PathBuf, + exec: Option<&PathBuf>, + expect_ok: bool, + ) -> Result { + let mut cmd = base.clone(); + Self::replace_cmd(&mut cmd, tmp_dir, input, output, exec); let mut r = process::Command::new(&cmd[0]); r.args(&cmd[1..]); + let done = r.output()?; + let code = done.status.code(); + match code { + Some(0) => return Ok(true), + status => { + if let Some(x) = status { + if x == 1 && !expect_ok { + return Ok(false); + } + } + let stderr_str = String::from_utf8_lossy(&done.stderr); + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("process exit with {:?}\n{}", status, stderr_str), + ) + .into()); + } + }; } pub fn run_test_schedule<'a>(&mut self, sched: &TestSchedule<'a>) -> Result<(), Error> { @@ -100,6 +163,83 @@ impl TestScheduler { let (tmp_dir, input, output) = self.create_input_file(sched, instance)?; + // build test + self.exec_cmd( + &sched.runner.build_command, + &tmp_dir, + &input, + &output, + None, + true, + )?; + + let exec = output; + + let output = tmp_dir.join("output.bin"); + + let input_binary = sched.input.binary.clone().into(); + + // run test + let status = self.exec_cmd( + &sched.runner.run_command, + &tmp_dir, + &input_binary, + &output, + Some(&exec), + false, + )?; + + let expect = !sched.input.failure_case; + + if status != expect { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("test failed: expect {} but got {}", expect, status), + ) + .into()); + } + + let input_binary = self.read_input_binary(&input_binary.to_string_lossy())?; // check input is valid + let output = fs::read(&output)?; // check output is valid + + let min_size = if input_binary.len() < output.len() { + input_binary.len() + } else { + output.len() + }; + + let mut diff = Vec::new(); + + for i in 0..min_size { + if input_binary[i] != output[i] { + diff.push((i, Some(input_binary[i]), Some(output[i]))); + } + } + + if input_binary.len() != output.len() { + if input_binary.len() > output.len() { + diff.push((output.len(), None, Some(output[output.len()]))); + } else { + diff.push(( + input_binary.len(), + Some(input_binary[input_binary.len()]), + None, + )); + } + } + + if !diff.is_empty() { + eprintln!("test failed: input and output is different"); + for (i, a, b) in diff { + eprintln!("{}: {:02x?} != {:02x?}", i, a, b); + } + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "test failed: input and output is different", + ) + .into()); + } + Ok(()) } } @@ -192,5 +332,9 @@ fn main() -> Result<(), Error> { } } } + let mut scheduler = TestScheduler::new(); + for s in sched { + scheduler.run_test_schedule(&s)?; + } Ok(()) } From 428035b5a67b70ac07414f72d1cc87fcc8f75cc3 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:23:09 +0900 Subject: [PATCH 06/50] fix typo --- src/tool/json2c/main.cpp | 2 +- src/tool/json2cpp2/main.cpp | 2 +- testutil/src/main.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tool/json2c/main.cpp b/src/tool/json2c/main.cpp index 8c76a84e..36e3ee44 100644 --- a/src/tool/json2c/main.cpp +++ b/src/tool/json2c/main.cpp @@ -67,7 +67,7 @@ int generate_c(const Flags& flags, brgen::request::GenerateSource& req, std::sha field("structs", g.struct_names); field("line_map", g.line_map); } - send_source(req.id, std::move(s.out()), req.name + ".c.map.json"); + send_source(req.id, std::move(s.out()), req.name + ".c.json"); } send_end_response(req.id); return 0; diff --git a/src/tool/json2cpp2/main.cpp b/src/tool/json2cpp2/main.cpp index 11afc327..c1f84ea6 100644 --- a/src/tool/json2cpp2/main.cpp +++ b/src/tool/json2cpp2/main.cpp @@ -58,7 +58,7 @@ int cpp_generate(const Flags& flags, brgen::request::GenerateSource& req, std::s field("structs", g.struct_names); field("line_map", g.line_map); } - send_source(req.id, std::move(s.out()), req.name + ".hpp.map.json"); + send_source(req.id, std::move(s.out()), req.name + ".hpp.json"); } send_end_response(req.id); return 0; diff --git a/testutil/src/main.rs b/testutil/src/main.rs index 59746ec6..efd93523 100644 --- a/testutil/src/main.rs +++ b/testutil/src/main.rs @@ -54,7 +54,7 @@ impl TestScheduler { if let Some(x) = self.input_binaries.get(path) { return Ok(x.clone()); } else { - let t = fs::read_to_string(path)?; + let t = fs::read(path)?; self.input_binaries.insert(path.to_string(), t); Ok(self.input_binaries.get(path).unwrap().clone()) } From 979e812b5f90d3d09d4e7f8fda43967fd1e044de Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Thu, 25 Apr 2024 19:19:01 +0900 Subject: [PATCH 07/50] add golang stdin_stream --- ast2go/request/response.go | 54 +++++++++++++++--- brgen.json | 2 +- build.bat | 9 +-- build.sh | 13 +++-- src/CMakeLists.txt | 79 +++++++++++++++++++++++--- src/tool/json2go/generate.go | 9 +-- src/tool/json2go/main.go | 107 +++++++++++++++++++++++++++-------- testutil/src/main.rs | 25 ++++---- 8 files changed, 234 insertions(+), 64 deletions(-) diff --git a/ast2go/request/response.go b/ast2go/request/response.go index e19bb2a7..b78d3392 100644 --- a/ast2go/request/response.go +++ b/ast2go/request/response.go @@ -7,8 +7,22 @@ type RequestStream struct { w io.Writer } -func NewRequestStream(r io.Reader, w io.Writer) *RequestStream { - return &RequestStream{r: r, w: w} +func NewRequestStream(r io.Reader, w io.Writer) (*RequestStream, error) { + h := &RequestStream{r: r, w: w} + if err := h.handshake(); err != nil { + return nil, err + } + return h, nil +} + +func (rs *RequestStream) handshake() error { + resp := &GeneratorResponseHeader{Version: 1} + enc := resp.MustEncode() + if _, err := rs.w.Write(enc); err != nil { + return err + } + hdr := &GeneratorRequestHeader{} + return hdr.Read(rs.r) } func (rs *RequestStream) Receive() (*GenerateSource, error) { @@ -20,11 +34,37 @@ func (rs *RequestStream) Receive() (*GenerateSource, error) { return src, nil } -func (rs *RequestStream) Respond(src *SourceCode) error { - enc, err := src.Encode() - if err != nil { - return err +func (rs *RequestStream) respond(id uint64, status ResponseStatus, name string, code []byte, warn string) error { + src := &SourceCode{ + ID: id, + Status: status, + } + src.SetName([]byte(name)) + src.SetCode(code) + src.SetErrorMessage([]byte(warn)) + resp := &Response{ + Type: ResponseType_Code, + } + resp.SetSource(*src) + enc := resp.MustEncode() + _, err := rs.w.Write(enc) + return err +} + +func (rs *RequestStream) RespondSource(id uint64, name string, code []byte, warn string) error { + return rs.respond(id, ResponseStatus_Ok, name, code, warn) +} + +func (rs *RequestStream) RespondError(id uint64, err string) error { + return rs.respond(id, ResponseStatus_Error, "", nil, err) +} + +func (rs *RequestStream) RespondEnd(id uint64) error { + resp := &Response{ + Type: ResponseType_EndOfCode, } - _, err = rs.w.Write(enc) + resp.SetEnd(EndOfCode{ID: id}) + enc := resp.MustEncode() + _, err := rs.w.Write(enc) return err } diff --git a/brgen.json b/brgen.json index 15316547..cb5858bc 100644 --- a/brgen.json +++ b/brgen.json @@ -53,7 +53,7 @@ "args": ["-as-code-block", "-format"] }, { - "generator": "./src/tool/json2rust/target/debug/json2rust", + "generator": "./tool/json2rust", "output_dir": "./ignore/example/rust/", "ignore_missing": true }, diff --git a/build.bat b/build.bat index b3f4fc69..c2b699ee 100644 --- a/build.bat +++ b/build.bat @@ -1,6 +1,7 @@ @echo off setlocal set S2J_LIB=1 +set BRGEN_RUST_ENABLED=1 set BUILD_MODE=%1 set BUILD_TYPE=%2 set FUTILS_DIR=%3 @@ -41,10 +42,10 @@ rem ninja -C ./built/%BUILD_MODE%/%BUILD_TYPE% ninja -C ./built/%BUILD_MODE%/%BUILD_TYPE% install if "%BUILD_MODE%" == "wasm-em" ( - cd ./src/tool/json2rust - wasm-pack build --target web - copy ..\..\..\LICENSE .\pkg\LICENSE - cd ../../../ +rem cd ./src/tool/json2rust +rem wasm-pack build --target web +rem copy ..\..\..\LICENSE .\pkg\LICENSE +rem cd ../../../ cd ./web/dev call tsc call webpack diff --git a/build.sh b/build.sh index 34614e9b..c8d9c3af 100644 --- a/build.sh +++ b/build.sh @@ -4,6 +4,8 @@ if [ "$(uname)" != "Darwin" ]; then export S2J_LIB=1 fi +export BRGEN_RUST_ENABLED=1 + BUILD_MODE=$1 BUILD_TYPE=$2 @@ -48,16 +50,16 @@ if [ $BUILD_MODE = "wasm-em" ];then else cmake -D CMAKE_CXX_COMPILER=$CXX_COMPILER -D CMAKE_C_COMPILER=$C_COMPILER -G Ninja -D CMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -D CMAKE_BUILD_TYPE=$BUILD_TYPE -S . -B ./built/$BUILD_MODE/$BUILD_TYPE fi -ninja -C ./built/$BUILD_MODE/$BUILD_TYPE +#ninja -C ./built/$BUILD_MODE/$BUILD_TYPE ninja -C ./built/$BUILD_MODE/$BUILD_TYPE install if [ $BUILD_MODE = "wasm-em" ]; then unset GOOS unset GOARCH -cd ./src/tool/json2rust -wasm-pack build --target web -cp ../../../LICENSE ./pkg -cd ../../.. +#cd ./src/tool/json2rust +#wasm-pack build --target web +#cp ../../../LICENSE ./pkg +#cd ../../.. cd ./web/dev tsc webpack @@ -68,3 +70,4 @@ fi unset FUTILS_DIR unset BUILD_MODE unset S2J_LIB +unset BRGEN_RUST_ENABLED \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 31fd9a50..f6021041 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,11 +1,10 @@ #license cmake_minimum_required(VERSION 3.22.1) -# find_package(Clang ) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tool") - +# Setting C++ targets add_executable(src2json "tool/src2json/src2json.cpp" "tool/src2json/entry.cpp" @@ -16,8 +15,6 @@ add_executable(src2json target_link_libraries(src2json futils) add_compile_definitions(BRGEN_VERSION="${PROJECT_VERSION}") - - add_executable(json2cpp "tool/json2cpp/main.cpp") target_link_libraries(json2cpp futils) @@ -33,6 +30,8 @@ target_link_libraries(json2vm futils) add_executable(json2ts "tool/json2ts/main.cpp" "tool/json2ts/generate.cpp") target_link_libraries(json2ts futils) + +# Setting Go targets if(WIN32) set(SUFFIX ".exe") elseif("$ENV{BUILD_MODE}" STREQUAL "wasm-em") @@ -44,7 +43,13 @@ add_custom_target(json2kaitai ALL COMMAND "${GO_COMPILER}" "build" "${CMAKE_GO_F add_custom_target(json2mermaid ALL COMMAND "${GO_COMPILER}" "build" "${CMAKE_GO_FLAGS}" "-o" "${CMAKE_BINARY_DIR}/tool/json2mermaid${SUFFIX}" "${CMAKE_SOURCE_DIR}/src/tool/json2mermaid" DEPENDS "${CMAKE_SOURCE_DIR}/src/tool/json2mermaid" "${CMAKE_SOURCE_DIR}/ast2go") + + + +# Setting Wasm + if("$ENV{BUILD_MODE}" STREQUAL "wasm-em") +# Setting C++ to Wasm targets install(TARGETS src2json DESTINATION "lib" OPTIONAL) install(FILES "${CMAKE_BINARY_DIR}/tool/src2json.wasm" DESTINATION "lib" OPTIONAL) install(TARGETS json2cpp DESTINATION "lib" OPTIONAL) @@ -58,20 +63,58 @@ install(FILES "${CMAKE_BINARY_DIR}/tool/json2vm.wasm" DESTINATION "lib" OPTIONAL install(TARGETS json2ts DESTINATION "lib" OPTIONAL) install(FILES "${CMAKE_BINARY_DIR}/tool/json2ts.wasm" DESTINATION "lib" OPTIONAL) +# Setting Go to Wasm targets install(FILES "${CMAKE_BINARY_DIR}/tool/json2go.wasm" DESTINATION "lib" OPTIONAL) install(FILES "${CMAKE_BINARY_DIR}/tool/json2kaitai.wasm" DESTINATION "lib" OPTIONAL) file(TO_CMAKE_PATH "$ENV{WASMEXEC_FILE}" WASMEXEC) install(FILES "${WASMEXEC}" RENAME "go_wasm_exec.js" DESTINATION "lib" OPTIONAL) install(CODE "file(APPEND \"${CMAKE_INSTALL_PREFIX}/lib/go_wasm_exec.js\" \"const Go = globalThis.Go;\n\")" OPTIONAL) install(CODE "file(APPEND \"${CMAKE_INSTALL_PREFIX}/lib/go_wasm_exec.js\" \"export {Go}\n\")" OPTIONAL) -return() + + +# Setting Rust to Wasm targets +if("$ENV{BRGEN_RUST_ENABLED}" STREQUAL "1") + add_custom_target( + json2rust ALL + COMMAND "wasm-bindgen" "build" "--target" "web" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/tool/json2rust + ) + add_custom_command( + TARGET json2rust POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/LICENSE" "${CMAKE_SOURCE_DIR}/src/tool/json2rust/pkg" + ) +endif() + +return() # Stop here when building for wasm +endif() + + +# Setting Rust targets for native + +if("$ENV{BRGEN_RUST_ENABLED}" STREQUAL "1") + + if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + set(RUST_RELEASE_OPTION "") + else() + set(RUST_RELEASE_OPTION "--release") + endif() + + add_custom_target( + json2rsut ALL + COMMAND "cargo" "build" "--bins" "${RUST_RELEASE_OPTION}" "--target-dir" "${CMAKE_BINARY_DIR}/tool/json2rust/target" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/tool/json2rust + ) + endif() +# Libraries + if(WIN32) set(CMAKE_SHARED_LIBRARY_PREFIX "lib") set(CMAKE_STATIC_LIBRARY_PREFIX "lib") endif() +# Setting shared library version src2json if(S2J_LIB) add_library(s2j SHARED "tool/src2json/src2json.cpp" @@ -85,6 +128,7 @@ target_link_libraries(s2j futils) install(TARGETS s2j DESTINATION "tool" OPTIONAL) endif() +# Setting network extension for src2json if(S2J_USE_NETWORK) target_sources(src2json PRIVATE "tool/src2json/network.cpp") target_link_libraries(src2json fnet fnetserv) @@ -96,6 +140,8 @@ add_compile_definitions(S2J_USE_NETWORK) endif() endif() + +# Setting C++ tools add_executable(ctobgn "tool/ctobgn/main.cpp") target_link_libraries(ctobgn futils) @@ -105,7 +151,7 @@ target_link_libraries(rfc2bgn futils) add_executable(hex2bin "tool/hex2bin/main.cpp") target_link_libraries(hex2bin futils) - +# Setting Go tools add_custom_target(brgen ALL COMMAND "${GO_COMPILER}" "build" "${CMAKE_GO_FLAGS}" "-o" "${CMAKE_BINARY_DIR}/tool/brgen${SUFFIX}" "${CMAKE_SOURCE_DIR}/src/tool/brgen" DEPENDS "${CMAKE_SOURCE_DIR}/src/tool/brgen" ) @@ -118,6 +164,7 @@ add_custom_target(gen_ast2csharp ALL COMMAND "${GO_COMPILER}" "build" "${CMAKE_G add_custom_target(gen_ast2mermaid ALL COMMAND "${GO_COMPILER}" "build" "${CMAKE_GO_FLAGS}" "-o" "${CMAKE_BINARY_DIR}/tool/gen_ast2mermaid${SUFFIX}" "${CMAKE_SOURCE_DIR}/src/tool/gen/ast2mermaid" DEPENDS "${CMAKE_SOURCE_DIR}/src/tool/gen/ast2mermaid" ) add_custom_target(gen_ast2dart ALL COMMAND "${GO_COMPILER}" "build" "${CMAKE_GO_FLAGS}" "-o" "${CMAKE_BINARY_DIR}/tool/gen_ast2dart${SUFFIX}" "${CMAKE_SOURCE_DIR}/src/tool/gen/ast2dart" DEPENDS "${CMAKE_SOURCE_DIR}/src/tool/gen/ast2dart" ) +# Copy dependency libraries (futils,fnet,fnetserv) if(WIN32) @@ -204,8 +251,10 @@ else() endif() endif() - +# Add test directory add_subdirectory(test) + +# Set RPATH for Unix-like systems if(UNIX) set_target_properties(src2json PROPERTIES INSTALL_RPATH "${CMAKE_SOURCE_DIR}/tool") set_target_properties(json2cpp PROPERTIES INSTALL_RPATH "${CMAKE_SOURCE_DIR}/tool") @@ -221,13 +270,17 @@ if(S2J_LIB) set_target_properties(s2j PROPERTIES INSTALL_RPATH "${CMAKE_SOURCE_DIR}/tool") endif() endif() + +# Set Install path for C++ targets + install(TARGETS src2json json2cpp json2cpp2 json2c json2vm json2ts ctobgn rfc2bgn hex2bin DESTINATION tool OPTIONAL) + +# Set Install path for Go targets install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/brgen${SUFFIX}" DESTINATION tool OPTIONAL) install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/json2go${SUFFIX}" DESTINATION tool OPTIONAL) install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/json2kaitai${SUFFIX}" DESTINATION tool OPTIONAL) install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/json2mermaid${SUFFIX}" DESTINATION tool OPTIONAL) - install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/gen_ast2go${SUFFIX}" DESTINATION tool OPTIONAL) install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/gen_ast2ts${SUFFIX}" DESTINATION tool OPTIONAL) install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/gen_ast2rust${SUFFIX}" DESTINATION tool OPTIONAL) @@ -236,3 +289,13 @@ install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/gen_ast2c${SUFFIX}" DESTINATION tool install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/gen_ast2csharp${SUFFIX}" DESTINATION tool OPTIONAL) install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/gen_ast2mermaid${SUFFIX}" DESTINATION tool OPTIONAL) install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/gen_ast2dart${SUFFIX}" DESTINATION tool OPTIONAL) + +# Set Install path for Rust targets +if("$ENV{BRGEN_RUST_ENABLED}" STREQUAL "1") + if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + set(RUST_TARGET_DIR "debug") + else() + set(RUST_TARGET_DIR "release") + endif() + install(PROGRAMS "${CMAKE_SOURCE_DIR}/src/tool/json2rust/target/${RUST_TARGET_DIR}/json2rust${SUFFIX}" DESTINATION tool OPTIONAL) +endif() diff --git a/src/tool/json2go/generate.go b/src/tool/json2go/generate.go index 882f9387..d3f9b7da 100644 --- a/src/tool/json2go/generate.go +++ b/src/tool/json2go/generate.go @@ -15,8 +15,9 @@ import ( "github.com/on-keyday/brgen/ast2go/gen" ) -var f = flag.Bool("s", false, "tell spec of json2go") -var filename = flag.String("f", "", "file to parse") +var spec = flag.Bool("s", false, "tell spec of json2go") +var filename = flag.Bool("f", false, "arg is filename") +var legacyStdin = flag.Bool("legacy-stdin", false, "use legacy stdin") // var usePut = flag.Bool("use-put", false, "use PutUintXXX instead of AppendUintXXX") var decodeReturnsLen = flag.Bool("decode-returns-len", true, "func Decode returns length of read bytes") @@ -35,8 +36,8 @@ func init() { } func ResetFlag() { - *f = false - *filename = "" + *spec = false + *filename = false //*usePut = false *decodeReturnsLen = true *useMustEncode = true diff --git a/src/tool/json2go/main.go b/src/tool/json2go/main.go index 7e8bd740..89e23212 100644 --- a/src/tool/json2go/main.go +++ b/src/tool/json2go/main.go @@ -3,56 +3,115 @@ package main import ( + "bytes" "encoding/json" "flag" "fmt" + "io" "os" ast2go "github.com/on-keyday/brgen/ast2go/ast" + "github.com/on-keyday/brgen/ast2go/request" ) +func generate(r io.Reader, out func(data []byte, warn error), errOut func(err error)) { + file := ast2go.AstFile{} + err := json.NewDecoder(r).Decode(&file) + if err != nil { + errOut(err) + return + } + g := NewGenerator() + + src, err := g.GenerateAndFormat(&file) + if err != nil && src == nil { + errOut(err) + } + if src != nil { + out(src, err) + } +} + +func streamingMode() { + stream, err := request.NewRequestStream(os.Stdin, os.Stdout) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for { + req, err := stream.Receive() + if err != nil { + break + } + r := bytes.NewReader(req.JsonText) + go func() { + generate(r, func(data []byte, warn error) { + stream.RespondSource(req.ID, string(req.Name)+".go", data, err.Error()) + }, func(err error) { + stream.RespondError(req.ID, err.Error()) + }) + }() + } +} + +var exitcode int + func main() { + defer os.Exit(exitcode) flag.Parse() - if *f { - fmt.Println(`{ + if *spec { + if *filename { + fmt.Println(`{ + "langs" : ["go"], + "input" : "file", + "suffix" : [".go"] + }`) + } else if *legacyStdin { + fmt.Println(`{ "langs" : ["go"], "input" : "stdin", "suffix" : [".go"] }`) + } else { + fmt.Println(`{ + "langs" : ["go"], + "input" : "stdin_stream" + }`) + } return } - file := ast2go.AstFile{} - if *filename != "" { - f, err := os.Open(*filename) - if err != nil { - fmt.Fprintln(os.Stderr, err) + + if !*filename && !*legacyStdin { + streamingMode() + return + } + + var f *os.File + if *filename { + if flag.NArg() < 1 { + fmt.Fprintln(os.Stderr, "no filename") os.Exit(1) return } - defer f.Close() - err = json.NewDecoder(f).Decode(&file) + var err error + f, err = os.Open(flag.Arg(0)) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) return } + defer f.Close() } else { - err := json.NewDecoder(os.Stdin).Decode(&file) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - return - } + f = os.Stdin } - g := NewGenerator() - src, err := g.GenerateAndFormat(&file) - if err != nil { - fmt.Fprintln(os.Stderr, err) - if src == nil { - os.Exit(1) + generate(f, func(src []byte, warn error) { + if warn != nil { + fmt.Fprintln(os.Stderr, warn) } - } - - os.Stdout.Write(src) + os.Stdout.Write(src) + }, func(err error) { + fmt.Fprintln(os.Stderr, err) + exitcode = 1 + }) } diff --git a/testutil/src/main.rs b/testutil/src/main.rs index efd93523..79993f80 100644 --- a/testutil/src/main.rs +++ b/testutil/src/main.rs @@ -229,15 +229,11 @@ impl TestScheduler { } if !diff.is_empty() { - eprintln!("test failed: input and output is different"); + let mut debug = format!("test failed: input and output is different\n"); for (i, a, b) in diff { - eprintln!("{}: {:02x?} != {:02x?}", i, a, b); + debug += &format!("{}: {:02x?} != {:02x?}\n", i, a, b); } - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "test failed: input and output is different", - ) - .into()); + return Err(std::io::Error::new(std::io::ErrorKind::Other, debug).into()); } Ok(()) @@ -272,7 +268,6 @@ fn main() -> Result<(), Error> { } has_input = false; has_output = false; - has_exec = false; for cmd in &runner.run_command { if cmd == "$INPUT" { has_input = true; @@ -307,7 +302,12 @@ fn main() -> Result<(), Error> { if file.suffix != ".json" { continue; } - let ext = file.base.split('.').last().unwrap(); + let split = file.base.split('.').collect::>(); + if split.len() != 2 { + continue; + } + let base = split[0]; + let ext = split[1]; let runner = match ext_set.get(ext) { Some(x) => x, None => continue, @@ -323,7 +323,7 @@ fn main() -> Result<(), Error> { } }; for s in &data.structs { - if let Some(input) = file_format_list.get(&(file.base.clone(), s.clone())) { + if let Some(input) = file_format_list.get(&(base.to_string(), s.clone())) { sched.push(TestSchedule { runner: runner, input: input, @@ -334,7 +334,10 @@ fn main() -> Result<(), Error> { } let mut scheduler = TestScheduler::new(); for s in sched { - scheduler.run_test_schedule(&s)?; + match scheduler.run_test_schedule(&s) { + Ok(()) => println!("test passed: {:?}", s.file.into_path() + &s.input.format_name), + Err(x) => println!("test failed: {:?} {:?}", s.file.into_path() + &s.input.format_name, x), + } } Ok(()) } From 5f5f3ba9215bddc8462fd64b643c172bf2f7e6d7 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Thu, 25 Apr 2024 19:42:03 +0900 Subject: [PATCH 08/50] fix --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f6021041..867cdd91 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -76,7 +76,7 @@ install(CODE "file(APPEND \"${CMAKE_INSTALL_PREFIX}/lib/go_wasm_exec.js\" \"expo if("$ENV{BRGEN_RUST_ENABLED}" STREQUAL "1") add_custom_target( json2rust ALL - COMMAND "wasm-bindgen" "build" "--target" "web" + COMMAND "wasm-pack" "build" "--target" "web" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/tool/json2rust ) add_custom_command( From 1145d6519026917941b213b1d565b070af5d9e7b Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 01:00:13 +0900 Subject: [PATCH 09/50] fix json2mermaid lib --- ast2go/request/response.go | 46 +++++++++++++++- src/tool/common/send.h | 2 +- src/tool/json2go/main.go | 29 +++++----- src/tool/json2mermaid/main.go | 99 +++++++++++++++++++++++++---------- test-brgen.json | 20 +++++++ 5 files changed, 152 insertions(+), 44 deletions(-) diff --git a/ast2go/request/response.go b/ast2go/request/response.go index b78d3392..aac52ae6 100644 --- a/ast2go/request/response.go +++ b/ast2go/request/response.go @@ -1,6 +1,10 @@ package request -import "io" +import ( + "fmt" + "io" + "runtime/debug" +) type RequestStream struct { r io.Reader @@ -68,3 +72,43 @@ func (rs *RequestStream) RespondEnd(id uint64) error { _, err := rs.w.Write(enc) return err } + +type IDStream struct { + rs *RequestStream + id uint64 +} + +func (rs *IDStream) RespondSource(name string, code []byte, warn string) error { + return rs.rs.RespondSource(rs.id, name, code, warn) +} + +func (rs *IDStream) RespondError(err string) error { + return rs.rs.RespondError(rs.id, err) +} + +func Run(r io.Reader, w io.Writer, cb func(s *IDStream, req *GenerateSource)) error { + stream, err := NewRequestStream(r, w) + if err != nil { + return err + } + for { + req, err := stream.Receive() + if err != nil { + break + } + go func() { + defer stream.RespondEnd(req.ID) + defer func() { + if r := recover(); r != nil { + trace := debug.Stack() + stream.RespondError(req.ID, fmt.Sprintf("%v\n%s", r, trace)) + } + }() + cb(&IDStream{ + stream, + req.ID, + }, req) + }() + } + return nil +} diff --git a/src/tool/common/send.h b/src/tool/common/send.h index 79a24245..3c5d1a20 100644 --- a/src/tool/common/send.h +++ b/src/tool/common/send.h @@ -121,7 +121,7 @@ inline void read_stdin_requests(auto&& cb) { break; } if (s != futils::thread::ChanStateValue::enable) { - std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); continue; } cb(req); diff --git a/src/tool/json2go/main.go b/src/tool/json2go/main.go index 89e23212..22d088f6 100644 --- a/src/tool/json2go/main.go +++ b/src/tool/json2go/main.go @@ -33,24 +33,23 @@ func generate(r io.Reader, out func(data []byte, warn error), errOut func(err er } func streamingMode() { - stream, err := request.NewRequestStream(os.Stdin, os.Stdout) + handler := func(stream *request.IDStream, req *request.GenerateSource) { + r := bytes.NewReader(req.JsonText) + generate(r, func(data []byte, warn error) { + err := "" + if warn != nil { + err = warn.Error() + } + stream.RespondSource(string(req.Name)+".go", data, err) + }, func(err error) { + stream.RespondError(err.Error()) + }) + } + err := request.Run(os.Stdin, os.Stdout, handler) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) - } - for { - req, err := stream.Receive() - if err != nil { - break - } - r := bytes.NewReader(req.JsonText) - go func() { - generate(r, func(data []byte, warn error) { - stream.RespondSource(req.ID, string(req.Name)+".go", data, err.Error()) - }, func(err error) { - stream.RespondError(req.ID, err.Error()) - }) - }() + return } } diff --git a/src/tool/json2mermaid/main.go b/src/tool/json2mermaid/main.go index 6df18607..08738c01 100644 --- a/src/tool/json2mermaid/main.go +++ b/src/tool/json2mermaid/main.go @@ -12,10 +12,12 @@ import ( ast "github.com/on-keyday/brgen/ast2go/ast" "github.com/on-keyday/brgen/ast2go/gen" + "github.com/on-keyday/brgen/ast2go/request" ) var spec = flag.Bool("s", false, "spec of this generator") -var fileName = flag.String("f", "", "file to generate") +var fileName = flag.Bool("f", false, "file to generate") +var legacyStdin = flag.Bool("stdin", false, "use stdin") var asCodeBlock = flag.Bool("as-code-block", false, "output as code block") var format = flag.Bool("format", false, "output only format and enum") @@ -185,47 +187,90 @@ func traceFormat(out io.Writer, prog *ast.Program) { })) } +func generate(out io.Writer, in io.Reader) error { + file := &ast.AstFile{} + err := json.NewDecoder(in).Decode(file) + if err != nil { + return err + } + prog, err := ast.ParseAST(file.Ast) + if err != nil { + return err + } + if *asCodeBlock { + fmt.Fprintf(out, "```mermaid\n") + defer fmt.Fprintf(out, "```\n") + } + if *format { + traceFormat(out, prog) + return nil + } + diagram(out, prog) + return nil +} + +func streamMode() { + err := request.Run(os.Stdin, os.Stdout, func(stream *request.IDStream, req *request.GenerateSource) { + r := bytes.NewReader(req.JsonText) + w := bytes.NewBuffer(nil) + err := generate(w, r) + if err != nil { + stream.RespondError(err.Error()) + } else { + stream.RespondSource(string(req.Name)+".md", w.Bytes(), "") + } + }) + if err != nil { + log.Fatal(err) + } +} + func main() { flag.Parse() if *spec { - fmt.Fprintf(os.Stdout, ` + if *legacyStdin { + fmt.Fprintf(os.Stdout, ` { "input": "stdin", "langs": ["mermaid"], "suffix": [".md"] } `) + } else if *fileName { + fmt.Fprintf(os.Stdout, ` + { + "input": "file", + "langs": ["mermaid"], + "suffix": [".md"] + } + `) + } else { + fmt.Fprintf(os.Stdout, ` + { + "input": "stdin_stream", + "langs": ["mermaid"] + } + `) + } return } - file := &ast.AstFile{} - if *fileName == "" { - err := json.NewDecoder(os.Stdin).Decode(file) - if err != nil { - log.Fatal(err) - } + if !*legacyStdin && !*fileName { + streamMode() + return + } + + var f *os.File + if !*fileName { + f = os.Stdin } else { - f, err := os.Open(*fileName) - if err != nil { - log.Fatal(err) + if flag.NArg() != 1 { + log.Fatal("no filename") } - defer f.Close() - err = json.NewDecoder(f).Decode(file) + f, err := os.Open(flag.Arg(0)) if err != nil { log.Fatal(err) } + defer f.Close() } - prog, err := ast.ParseAST(file.Ast) - if err != nil { - log.Fatal(err) - } - out := os.Stdout - if *asCodeBlock { - fmt.Fprintf(out, "```mermaid\n") - defer fmt.Fprintf(out, "```\n") - } - if *format { - traceFormat(out, prog) - return - } - diagram(out, prog) + generate(os.Stdout, f) } diff --git a/test-brgen.json b/test-brgen.json index 89aba8e8..11711ae4 100644 --- a/test-brgen.json +++ b/test-brgen.json @@ -15,6 +15,26 @@ }, "test_info_output": "./ignore/example/test_info.json", "output": [ + { + "generator": "./tool/json2go", + "output_dir": "./ignore/example/go/", + "args": [ + "-map-word", + "Id=ID", + "-map-word", + "IDentifier=Identifier", + "-map-word", + "Tcpsegment=TCPSegment", + "-map-word", + "Udpdatagram=UDPDatagram", + "-map-word", + "Udpheader=UDPHeader", + "-map-word", + "icmp=ICMP" + ] + } + ], + "garbage": [ { "generator": "./tool/json2cpp2", "output_dir": "./ignore/example/cpp2/", From aae2b07ae5c60452a271ae032618d0372ded7347 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 01:02:13 +0900 Subject: [PATCH 10/50] move into tool --- {testutil => src/tool/cmptest}/Cargo.lock | 0 {testutil => src/tool/cmptest}/Cargo.toml | 0 {testutil => src/tool/cmptest}/src/main.rs | 0 {testutil => src/tool/cmptest}/src/testutil.rs | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {testutil => src/tool/cmptest}/Cargo.lock (100%) rename {testutil => src/tool/cmptest}/Cargo.toml (100%) rename {testutil => src/tool/cmptest}/src/main.rs (100%) rename {testutil => src/tool/cmptest}/src/testutil.rs (100%) diff --git a/testutil/Cargo.lock b/src/tool/cmptest/Cargo.lock similarity index 100% rename from testutil/Cargo.lock rename to src/tool/cmptest/Cargo.lock diff --git a/testutil/Cargo.toml b/src/tool/cmptest/Cargo.toml similarity index 100% rename from testutil/Cargo.toml rename to src/tool/cmptest/Cargo.toml diff --git a/testutil/src/main.rs b/src/tool/cmptest/src/main.rs similarity index 100% rename from testutil/src/main.rs rename to src/tool/cmptest/src/main.rs diff --git a/testutil/src/testutil.rs b/src/tool/cmptest/src/testutil.rs similarity index 100% rename from testutil/src/testutil.rs rename to src/tool/cmptest/src/testutil.rs From 53226cd66fcdcbc9a823afb305648f50ff17e621 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:13:42 +0900 Subject: [PATCH 11/50] add gzip --- .vscode/settings.json | 2 +- CMakeLists.txt | 4 ++ example/fs.txt | 20 ++++++++++ example/gzip.bgn | 60 ++++++++++++++++++++++++++++++ src/CMakeLists.txt | 7 ++++ src/tool/cmptest/Cargo.lock | 24 ++++++------ src/tool/cmptest/Cargo.toml | 4 +- web/doc/content/docs/exec_model.md | 29 +++++++++++++++ 8 files changed, 135 insertions(+), 15 deletions(-) create mode 100644 example/gzip.bgn diff --git a/.vscode/settings.json b/.vscode/settings.json index 7c246f9e..4bb291a9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -148,7 +148,7 @@ ".\\ast2rust\\Cargo.toml", ".\\src\\tool\\json2rust\\Cargo.toml", ".\\src\\tool\\json2llvm\\Cargo.toml", - ".\\testutil\\Cargo.toml", + ".\\src\\tool\\cmptest\\Cargo.toml", ], "brgen-lsp.src2json" :"./tool/src2json.exe", //"go.buildTags": "linux", diff --git a/CMakeLists.txt b/CMakeLists.txt index 951ed2f7..718c7a4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(NOT "$ENV{FUTILS_TARGET_TRIPLE}" STREQUAL "") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target=$ENV{FUTILS_TARGET_TRIPLE}") endif() + set(CMAKE_CXX_FLAGS "-fuse-ld=lld") endif() if(UNIX) @@ -16,6 +17,9 @@ set(CMAKE_EXE_LINKER_FLAGS "-lc++abi") endif() endif() + + + set(CMAKE_CXX_STANDARD 20) diff --git a/example/fs.txt b/example/fs.txt index dd0a745b..4898c519 100644 --- a/example/fs.txt +++ b/example/fs.txt @@ -426,4 +426,24 @@ enum Type { // others are reserved experimental_1 = 253, experimental_2 = 255, + }; + + enum OS { + fat = 0, + amiga = 1, + vms = 2, + unix_ = 3, + vm_cms = 4, + atari = 5, + hpfs = 6, + machintosh = 7, + z_system = 8, + cp_m = 9, + tops_20 = 10, + ntfs = 11, + qdos = 12, + acorn_risc_os = 13, + vfat = 14, + mvs = 15, + beos = 16, }; \ No newline at end of file diff --git a/example/gzip.bgn b/example/gzip.bgn new file mode 100644 index 00000000..58ba0317 --- /dev/null +++ b/example/gzip.bgn @@ -0,0 +1,60 @@ +config.url = "https://www.rfc-editor.org/rfc/rfc1952" +input.endian = config.endian.little +input.bit_order = config.bit_order.lsb + +format GZipHeader: + magic :"\x1f\x8b" + method :CompressionMethod + flags :GZipFlag + mtime :u32 + xfl :u8 + os :OS + if flags.extra ==1: + extra :GZipExtra + if flags.fname ==1: + fname :CString + if flags.fcomment ==1: + fcomment :CString + if flags.crc ==1: + crc16 :u16 + +format GZipExtra: + xlen :u16 + data :[xlen]u8 + +format CString: + data :[..]u8 + :"\x00" + +format GZipFlag: + text :u1 + crc :u1 + extra :u1 + fname :u1 + fcomment :u1 + reserved :u3 + reserved == 0 + +enum CompressionMethod: + :u8 + deflate = 8 + +enum OS: + :u8 + fat = 0 + amiga = 1 + vms = 2 + unix_ = 3 + vm_cms = 4 + atari = 5 + hpfs = 6 + machintosh = 7 + z_system = 8 + cp_m = 9 + tops_20 = 10 + ntfs = 11 + qdos = 12 + acorn_risc_os = 13 + vfat = 14 + mvs = 15 + beos = 16 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 867cdd91..c5d887e9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,6 +105,12 @@ if("$ENV{BRGEN_RUST_ENABLED}" STREQUAL "1") WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/tool/json2rust ) + add_custom_target( + cmptest ALL + COMMAND "cargo" "build" "--bins" "${RUST_RELEASE_OPTION}" "--target-dir" "${CMAKE_BINARY_DIR}/tool/cmptest/target" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/tool/cmptest + ) + endif() # Libraries @@ -298,4 +304,5 @@ if("$ENV{BRGEN_RUST_ENABLED}" STREQUAL "1") set(RUST_TARGET_DIR "release") endif() install(PROGRAMS "${CMAKE_SOURCE_DIR}/src/tool/json2rust/target/${RUST_TARGET_DIR}/json2rust${SUFFIX}" DESTINATION tool OPTIONAL) + install(PROGRAMS "${CMAKE_SOURCE_DIR}/src/tool/cmptest/target/${RUST_TARGET_DIR}/cmptest${SUFFIX}" DESTINATION tool OPTIONAL) endif() diff --git a/src/tool/cmptest/Cargo.lock b/src/tool/cmptest/Cargo.lock index 4233584b..8ff31353 100644 --- a/src/tool/cmptest/Cargo.lock +++ b/src/tool/cmptest/Cargo.lock @@ -114,6 +114,18 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "cmptest" +version = "0.1.0" +dependencies = [ + "ast2rust", + "clap", + "rand", + "regex", + "serde", + "serde_json", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -292,18 +304,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "testutil" -version = "0.1.0" -dependencies = [ - "ast2rust", - "clap", - "rand", - "regex", - "serde", - "serde_json", -] - [[package]] name = "unicode-ident" version = "1.0.12" diff --git a/src/tool/cmptest/Cargo.toml b/src/tool/cmptest/Cargo.toml index fee73625..f5182ad5 100644 --- a/src/tool/cmptest/Cargo.toml +++ b/src/tool/cmptest/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "testutil" +name = "cmptest" version = "0.1.0" edition = "2021" @@ -9,6 +9,6 @@ edition = "2021" clap = { version = "4.5.4", features = ["derive"] } serde = { version="1.0.196" ,features = ["derive"]} serde_json = "1.0.116" -ast2rust = {path ="../ast2rust" } +ast2rust = {path ="../../../ast2rust" } regex = "1.10.4" rand = "0.8.5" diff --git a/web/doc/content/docs/exec_model.md b/web/doc/content/docs/exec_model.md index d9d9e8df..ada9f93f 100644 --- a/web/doc/content/docs/exec_model.md +++ b/web/doc/content/docs/exec_model.md @@ -301,3 +301,32 @@ instance = { d: [ 0x0102, 0x0304, 0x0506, 0x0708 ] } ``` TODO(on-keyday): 続きを書く + +# ビットフィールド + +本言語ではビットフィールドが特別扱いされないという事になっている。 +まあ筆者も他の類似ツールが`bitfield(16) { n :1, reserved :15 }`みたいな書き方しているのや capnproto とかいうフォーマットのやつがこの言語の書き方では +うまく表せないなあと気づいたときに返るべきかと思ったりはしているが、 +現状こんな + +``` +format E: + a :u2 + b :u2 + d :u4 +``` + +なにも指定しないデフォルトの状態ではこれらは上位ビットから割り付けられ行く +マスク値で言うと a が 0xC0,b が 0x30,c が 0x0f である。 + +なおこの割り付け順序は以下のようにすると lsb からになる。 + +``` +format F: + input.bit_order = config.bit_order.lsb + a :u2 + b :u2 + d :u4 +``` + +この場合は From c03f2aec4ea941a95cbe607142f672962c2c8dcc Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:14:45 +0900 Subject: [PATCH 12/50] try fuse-ld=lld --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index c8d9c3af..bd1aefa5 100644 --- a/build.sh +++ b/build.sh @@ -1,8 +1,8 @@ #!/bin/bash # TODO(on-keyday): on macos, S2J_LIB=1 is not work well, so not set S2J_LIB -if [ "$(uname)" != "Darwin" ]; then - export S2J_LIB=1 -fi + +export S2J_LIB=1 + export BRGEN_RUST_ENABLED=1 From 9b0246ad8cef3936390a9e54db6ac3e9bec7d384 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:40:04 +0900 Subject: [PATCH 13/50] fix macos-latest install path for latest update --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd22f665..55e25b2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,8 +62,8 @@ jobs: brew install llvm - name: Build run: | - export FUTILS_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++ - export FUTILS_C_COMPILER=/usr/local/opt/llvm/bin/clang + export FUTILS_CXX_COMPILER=/opt/homebrew//opt/llvm/bin/clang++ + export FUTILS_C_COMPILER=/opt/homebrew/opt/llvm/bin/clang . build.sh native Release ./tool/src2json --version - name: Pack artifacts From 6681a5dcfb36a753e5158a5c8ecda7479b3f336e Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:45:44 +0900 Subject: [PATCH 14/50] setup go for mac --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55e25b2b..5d2b252b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,6 +57,11 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Set up Ninja uses: seanmiddleditch/gha-setup-ninja@8b297075da4cd2a5f1fd21fe011b499edf06e9d2 # master + - name: Set up Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version: ">=1.20" + - name: Install Homebrew Clang run: | brew install llvm From 61a8ccb13dd42454b40837652f23acfa0b445897 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:58:33 +0900 Subject: [PATCH 15/50] try again --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 718c7a4e..455b6e0f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(NOT "$ENV{FUTILS_TARGET_TRIPLE}" STREQUAL "") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target=$ENV{FUTILS_TARGET_TRIPLE}") endif() - set(CMAKE_CXX_FLAGS "-fuse-ld=lld") + set(CMAKE_CXX_FLAGS "-fuse-ld=ld.lld") endif() if(UNIX) From 5117c21c8cc8e812f6ecc5838af3b3e35f25c70f Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:06:13 +0900 Subject: [PATCH 16/50] how to fix? --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 455b6e0f..718c7a4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(NOT "$ENV{FUTILS_TARGET_TRIPLE}" STREQUAL "") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target=$ENV{FUTILS_TARGET_TRIPLE}") endif() - set(CMAKE_CXX_FLAGS "-fuse-ld=ld.lld") + set(CMAKE_CXX_FLAGS "-fuse-ld=lld") endif() if(UNIX) From 38b35c0bb134b0e070b225deebf43067c1169f8d Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:51:57 +0900 Subject: [PATCH 17/50] fix --- src/tool/cmptest/src/main.rs | 247 ++----------------------------- src/tool/cmptest/src/testutil.rs | 233 ++++++++++++++++++++++++++++- 2 files changed, 245 insertions(+), 235 deletions(-) diff --git a/src/tool/cmptest/src/main.rs b/src/tool/cmptest/src/main.rs index 79993f80..4d7ab9b1 100644 --- a/src/tool/cmptest/src/main.rs +++ b/src/tool/cmptest/src/main.rs @@ -1,20 +1,9 @@ -use std::{ - collections::HashMap, - env, - ffi::OsStr, - fs, os, - path::{Path, PathBuf}, - process, -}; +use std::{fs, path::Path}; use clap::{arg, Parser}; use serde::Deserialize; -use testutil::TestSchedule; +use testutil::{Error, TestSchedule}; mod testutil; -use rand::{ - self, - distributions::{Alphanumeric, DistString}, -}; #[derive(Parser)] struct Args { @@ -23,226 +12,9 @@ struct Args { #[arg(long, short('c'))] test_config_file: String, } -type Error = Box; - -struct TestScheduler { - template_files: HashMap, - tmpdir: Option, - input_binaries: HashMap>, -} - -impl TestScheduler { - fn new() -> Self { - Self { - template_files: HashMap::new(), - tmpdir: None, - input_binaries: HashMap::new(), - } - } - - fn read_template(&mut self, path: &str) -> Result { - if let Some(x) = self.template_files.get(path) { - return Ok(x.clone()); - } else { - let t = fs::read_to_string(path)?; - self.template_files.insert(path.to_string(), t); - Ok(self.template_files.get(path).unwrap().clone()) - } - } - - fn read_input_binary(&mut self, path: &str) -> Result, Error> { - if let Some(x) = self.input_binaries.get(path) { - return Ok(x.clone()); - } else { - let t = fs::read(path)?; - self.input_binaries.insert(path.to_string(), t); - Ok(self.input_binaries.get(path).unwrap().clone()) - } - } - - fn get_tmp_dir<'a>(&'a mut self) -> PathBuf { - if let Some(x) = self.tmpdir.as_ref() { - x.clone() - } else { - let dir = env::temp_dir(); - let mut rng = rand::thread_rng(); - let random_str = Alphanumeric.sample_string(&mut rng, 32); - let dir = dir.join(random_str); - self.tmpdir = Some(dir); - self.tmpdir.as_ref().unwrap().clone() - } - } - - fn prepare_content<'a>(&mut self, sched: &TestSchedule<'a>) -> Result { - // get template and replace with target - let template = self.read_template(&sched.runner.test_template)?; - let replace_with = &sched.runner.replace_struct_name; - let template = template.replace(replace_with, &sched.input.format_name); - let replace_with = &sched.runner.replace_file_name; - let path = format!("{}/{}", sched.file.dir, sched.file.base); - let instance = template.replace(replace_with, &path); - Ok(instance) - } - - fn create_input_file<'a>( - &mut self, - sched: &TestSchedule<'a>, - instance: String, - ) -> Result<(PathBuf, PathBuf, PathBuf), Error> { - let tmp_dir = self.get_tmp_dir(); - let tmp_dir = tmp_dir.join(&sched.file.base); - let tmp_dir = tmp_dir.join(&sched.input.format_name); - let tmp_dir = tmp_dir.join(&sched.file.suffix); - fs::create_dir_all(&tmp_dir)?; - let input_file = tmp_dir.join(&sched.runner.build_input_name); - let output_file = tmp_dir.join(&sched.runner.build_output_name); - fs::write(&input_file, instance)?; - Ok((tmp_dir, input_file, output_file)) - } - - fn replace_cmd( - cmd: &mut Vec, - tmp_dir: &PathBuf, - input: &PathBuf, - output: &PathBuf, - exec: Option<&PathBuf>, - ) { - for c in cmd { - if c == "$INPUT" { - *c = input.to_str().unwrap().to_string(); - } - if c == "$OUTPUT" { - *c = output.to_str().unwrap().to_string(); - } - if c == "$EXEC" { - if let Some(e) = exec { - *c = e.to_str().unwrap().to_string(); - } - } - if c == "$TMPDIR" { - *c = tmp_dir.to_str().unwrap().to_string(); - } - } - } - - fn exec_cmd<'a>( - &mut self, - base: &Vec, - tmp_dir: &PathBuf, - input: &PathBuf, - output: &PathBuf, - exec: Option<&PathBuf>, - expect_ok: bool, - ) -> Result { - let mut cmd = base.clone(); - Self::replace_cmd(&mut cmd, tmp_dir, input, output, exec); - let mut r = process::Command::new(&cmd[0]); - r.args(&cmd[1..]); - let done = r.output()?; - let code = done.status.code(); - match code { - Some(0) => return Ok(true), - status => { - if let Some(x) = status { - if x == 1 && !expect_ok { - return Ok(false); - } - } - let stderr_str = String::from_utf8_lossy(&done.stderr); - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("process exit with {:?}\n{}", status, stderr_str), - ) - .into()); - } - }; - } - - pub fn run_test_schedule<'a>(&mut self, sched: &TestSchedule<'a>) -> Result<(), Error> { - let instance = self.prepare_content(sched)?; - - let (tmp_dir, input, output) = self.create_input_file(sched, instance)?; - - // build test - self.exec_cmd( - &sched.runner.build_command, - &tmp_dir, - &input, - &output, - None, - true, - )?; - - let exec = output; - - let output = tmp_dir.join("output.bin"); - - let input_binary = sched.input.binary.clone().into(); - - // run test - let status = self.exec_cmd( - &sched.runner.run_command, - &tmp_dir, - &input_binary, - &output, - Some(&exec), - false, - )?; - - let expect = !sched.input.failure_case; - - if status != expect { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("test failed: expect {} but got {}", expect, status), - ) - .into()); - } - - let input_binary = self.read_input_binary(&input_binary.to_string_lossy())?; // check input is valid - let output = fs::read(&output)?; // check output is valid - - let min_size = if input_binary.len() < output.len() { - input_binary.len() - } else { - output.len() - }; - - let mut diff = Vec::new(); - - for i in 0..min_size { - if input_binary[i] != output[i] { - diff.push((i, Some(input_binary[i]), Some(output[i]))); - } - } - - if input_binary.len() != output.len() { - if input_binary.len() > output.len() { - diff.push((output.len(), None, Some(output[output.len()]))); - } else { - diff.push(( - input_binary.len(), - Some(input_binary[input_binary.len()]), - None, - )); - } - } - - if !diff.is_empty() { - let mut debug = format!("test failed: input and output is different\n"); - for (i, a, b) in diff { - debug += &format!("{}: {:02x?} != {:02x?}\n", i, a, b); - } - return Err(std::io::Error::new(std::io::ErrorKind::Other, debug).into()); - } - - Ok(()) - } -} - fn main() -> Result<(), Error> { let parsed = Args::parse(); - let test_config = fs::read_to_string(Path::new(&parsed.test_config_file))?; + let test_config =fs::read_to_string(Path::new (&parsed.test_config_file))?; let mut d1 = serde_json::Deserializer::from_str(&test_config); let test_config = testutil::TestConfig::deserialize(&mut d1)?; let mut ext_set = std::collections::HashMap::new(); @@ -332,11 +104,18 @@ fn main() -> Result<(), Error> { } } } - let mut scheduler = TestScheduler::new(); + let mut scheduler = testutil::TestScheduler::new(); for s in sched { match scheduler.run_test_schedule(&s) { - Ok(()) => println!("test passed: {:?}", s.file.into_path() + &s.input.format_name), - Err(x) => println!("test failed: {:?} {:?}", s.file.into_path() + &s.input.format_name, x), + Ok(()) => println!( + "test passed: {:?}", + s.file.into_path() + &s.input.format_name + ), + Err(x) => println!( + "test failed: {:?} {:?}", + s.file.into_path() + &s.input.format_name, + x + ), } } Ok(()) diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index cc5ffce8..f5c7a5c3 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -1,4 +1,17 @@ -use std::path::{Path, PathBuf}; + +use std::{ + collections::HashMap, + env, + ffi::OsStr, + fs, os, + path::{Path, PathBuf}, + process, +}; + +use rand::{ + self, + distributions::{Alphanumeric, DistString}, +}; use ast2rust::ast; use serde::{Deserialize, Serialize}; @@ -94,3 +107,221 @@ pub struct TestSchedule<'a> { pub runner: &'a TestRunner, pub file: &'a GeneratedFileInfo, } + + +pub type Error = Box; + +pub struct TestScheduler { + template_files: HashMap, + tmpdir: Option, + input_binaries: HashMap>, +} + +impl TestScheduler { + pub fn new() -> Self { + Self { + template_files: HashMap::new(), + tmpdir: None, + input_binaries: HashMap::new(), + } + } + + fn read_template(&mut self, path: &str) -> Result { + if let Some(x) = self.template_files.get(path) { + return Ok(x.clone()); + } else { + let t = fs::read_to_string(path)?; + self.template_files.insert(path.to_string(), t); + Ok(self.template_files.get(path).unwrap().clone()) + } + } + + fn read_input_binary(&mut self, path: &str) -> Result, Error> { + if let Some(x) = self.input_binaries.get(path) { + return Ok(x.clone()); + } else { + let t = fs::read(path)?; + self.input_binaries.insert(path.to_string(), t); + Ok(self.input_binaries.get(path).unwrap().clone()) + } + } + + fn get_tmp_dir<'a>(&'a mut self) -> PathBuf { + if let Some(x) = self.tmpdir.as_ref() { + x.clone() + } else { + let dir = env::temp_dir(); + let mut rng = rand::thread_rng(); + let random_str = Alphanumeric.sample_string(&mut rng, 32); + let dir = dir.join(random_str); + self.tmpdir = Some(dir); + self.tmpdir.as_ref().unwrap().clone() + } + } + + fn prepare_content<'a>(&mut self, sched: &TestSchedule<'a>) -> Result { + // get template and replace with target + let template = self.read_template(&sched.runner.test_template)?; + let replace_with = &sched.runner.replace_struct_name; + let template = template.replace(replace_with, &sched.input.format_name); + let replace_with = &sched.runner.replace_file_name; + let path = format!("{}/{}", sched.file.dir, sched.file.base); + let instance = template.replace(replace_with, &path); + Ok(instance) + } + + fn create_input_file<'a>( + &mut self, + sched: &TestSchedule<'a>, + instance: String, + ) -> Result<(PathBuf, PathBuf, PathBuf), Error> { + let tmp_dir = self.get_tmp_dir(); + let tmp_dir = tmp_dir.join(&sched.file.base); + let tmp_dir = tmp_dir.join(&sched.input.format_name); + let tmp_dir = tmp_dir.join(&sched.file.suffix); + fs::create_dir_all(&tmp_dir)?; + let input_file = tmp_dir.join(&sched.runner.build_input_name); + let output_file = tmp_dir.join(&sched.runner.build_output_name); + fs::write(&input_file, instance)?; + Ok((tmp_dir, input_file, output_file)) + } + + fn replace_cmd( + cmd: &mut Vec, + tmp_dir: &PathBuf, + input: &PathBuf, + output: &PathBuf, + exec: Option<&PathBuf>, + ) { + for c in cmd { + if c == "$INPUT" { + *c = input.to_str().unwrap().to_string(); + } + if c == "$OUTPUT" { + *c = output.to_str().unwrap().to_string(); + } + if c == "$EXEC" { + if let Some(e) = exec { + *c = e.to_str().unwrap().to_string(); + } + } + if c == "$TMPDIR" { + *c = tmp_dir.to_str().unwrap().to_string(); + } + } + } + + fn exec_cmd<'a>( + &mut self, + base: &Vec, + tmp_dir: &PathBuf, + input: &PathBuf, + output: &PathBuf, + exec: Option<&PathBuf>, + expect_ok: bool, + ) -> Result { + let mut cmd = base.clone(); + Self::replace_cmd(&mut cmd, tmp_dir, input, output, exec); + let mut r = process::Command::new(&cmd[0]); + r.args(&cmd[1..]); + let done = r.output()?; + let code = done.status.code(); + match code { + Some(0) => return Ok(true), + status => { + if let Some(x) = status { + if x == 1 && !expect_ok { + return Ok(false); + } + } + let stderr_str = String::from_utf8_lossy(&done.stderr); + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("process exit with {:?}\n{}", status, stderr_str), + ) + .into()); + } + }; + } + + pub fn run_test_schedule<'a>(&mut self, sched: &TestSchedule<'a>) -> Result<(), Error> { + let instance = self.prepare_content(sched)?; + + let (tmp_dir, input, output) = self.create_input_file(sched, instance)?; + + // build test + self.exec_cmd( + &sched.runner.build_command, + &tmp_dir, + &input, + &output, + None, + true, + )?; + + let exec = output; + + let output = tmp_dir.join("output.bin"); + + let input_binary = sched.input.binary.clone().into(); + + // run test + let status = self.exec_cmd( + &sched.runner.run_command, + &tmp_dir, + &input_binary, + &output, + Some(&exec), + false, + )?; + + let expect = !sched.input.failure_case; + + if status != expect { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("test failed: expect {} but got {}", expect, status), + ) + .into()); + } + + let input_binary = self.read_input_binary(&input_binary.to_string_lossy())?; // check input is valid + let output = fs::read(&output)?; // check output is valid + + let min_size = if input_binary.len() < output.len() { + input_binary.len() + } else { + output.len() + }; + + let mut diff = Vec::new(); + + for i in 0..min_size { + if input_binary[i] != output[i] { + diff.push((i, Some(input_binary[i]), Some(output[i]))); + } + } + + if input_binary.len() != output.len() { + if input_binary.len() > output.len() { + diff.push((output.len(), None, Some(output[output.len()]))); + } else { + diff.push(( + input_binary.len(), + Some(input_binary[input_binary.len()]), + None, + )); + } + } + + if !diff.is_empty() { + let mut debug = format!("test failed: input and output is different\n"); + for (i, a, b) in diff { + debug += &format!("{}: {:02x?} != {:02x?}\n", i, a, b); + } + return Err(std::io::Error::new(std::io::ErrorKind::Other, debug).into()); + } + + Ok(()) + } +} From ec14c4e4bd0de9da8091a9d2843031c60b1643d5 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:18:12 +0900 Subject: [PATCH 18/50] add LDFLAGS for mac --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5d2b252b..c5f5b4a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,6 +69,7 @@ jobs: run: | export FUTILS_CXX_COMPILER=/opt/homebrew//opt/llvm/bin/clang++ export FUTILS_C_COMPILER=/opt/homebrew/opt/llvm/bin/clang + export LDFLAGS="-L/opt/homebrew/opt/llvm/lib/c++ -Wl,-rpath,/opt/homebrew/opt/llvm/lib/c++" . build.sh native Release ./tool/src2json --version - name: Pack artifacts From c1c31be69277012341b947bd6caf39d88eaf9a52 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:37:29 +0900 Subject: [PATCH 19/50] adding typeinfo inspection --- .github/workflows/build.yml | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c5f5b4a3..480ece1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,15 +61,34 @@ jobs: uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: ">=1.20" - - name: Install Homebrew Clang run: | brew install llvm + + - name: Check Code + run: | + # from https://github.com/llvm/llvm-project/issues/77786 + cat << EOF | clang++ -xc++ - -c -o typeinfos.o && nm -omg typeinfos.o | c++filt | grep typeinfo + namespace __cxxabiv1 { + struct __fundamental_type_info { + virtual ~__fundamental_type_info(); + }; + + // This miraculously (compiler magic) emits the type_info's for: + // 1. all of the fundamental types + // 2. pointers to all of the fundamental types + // 3. pointers to all of the const fundamental types + __fundamental_type_info::~__fundamental_type_info() + { + } + } + EOF - name: Build run: | - export FUTILS_CXX_COMPILER=/opt/homebrew//opt/llvm/bin/clang++ - export FUTILS_C_COMPILER=/opt/homebrew/opt/llvm/bin/clang - export LDFLAGS="-L/opt/homebrew/opt/llvm/lib/c++ -Wl,-rpath,/opt/homebrew/opt/llvm/lib/c++" + PREFIX=$(brew --prefix llvm) + export FUTILS_CXX_COMPILER=$PREFIX/bin/clang++ + export FUTILS_C_COMPILER=$PREFIX/bin/clang + export LDFLAGS="-L$PREFIX/lib/c++ -Wl,-rpath,$PREFIX/lib/c++" . build.sh native Release ./tool/src2json --version - name: Pack artifacts From d77cbebccf4ae3b52ae5c8d4c49aa079454b964c Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:41:19 +0900 Subject: [PATCH 20/50] fix? --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 480ece1f..fcee4df0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -67,8 +67,9 @@ jobs: - name: Check Code run: | + PREFIX=$(brew --prefix llvm) # from https://github.com/llvm/llvm-project/issues/77786 - cat << EOF | clang++ -xc++ - -c -o typeinfos.o && nm -omg typeinfos.o | c++filt | grep typeinfo + cat << EOF | $PREFIX/bin/clang++ -xc++ - -c -o typeinfos.o && nm -omg typeinfos.o | c++filt | grep typeinfo namespace __cxxabiv1 { struct __fundamental_type_info { virtual ~__fundamental_type_info(); From 84b1344ee38274eacbc3e5a04c0b763cce205f10 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:52:49 +0900 Subject: [PATCH 21/50] fixing --- .github/workflows/build.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fcee4df0..7f3c12a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,21 +69,8 @@ jobs: run: | PREFIX=$(brew --prefix llvm) # from https://github.com/llvm/llvm-project/issues/77786 - cat << EOF | $PREFIX/bin/clang++ -xc++ - -c -o typeinfos.o && nm -omg typeinfos.o | c++filt | grep typeinfo - namespace __cxxabiv1 { - struct __fundamental_type_info { - virtual ~__fundamental_type_info(); - }; - - // This miraculously (compiler magic) emits the type_info's for: - // 1. all of the fundamental types - // 2. pointers to all of the fundamental types - // 3. pointers to all of the const fundamental types - __fundamental_type_info::~__fundamental_type_info() - { - } - } - EOF + LIBCPP=$PREFIX/lib/c++ + nm $LIBCPP | grep typeinfo - name: Build run: | PREFIX=$(brew --prefix llvm) From 4a064670b93909a50fa165fd4d5a77a99a532a99 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:55:10 +0900 Subject: [PATCH 22/50] fix? --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f3c12a0..616e8e6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,7 +70,7 @@ jobs: PREFIX=$(brew --prefix llvm) # from https://github.com/llvm/llvm-project/issues/77786 LIBCPP=$PREFIX/lib/c++ - nm $LIBCPP | grep typeinfo + nm $LIBCPP/libc++.1.dylib | grep typeinfo - name: Build run: | PREFIX=$(brew --prefix llvm) From 8fc606edc9e891831f409e87848a4ca347529c58 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 22:13:53 +0900 Subject: [PATCH 23/50] add cmptest util schema --- .github/workflows/build.yml | 1 + spec/brgen_cmptest_config_schema.json | 77 +++++++++++++++++++++++++++ src/CMakeLists.txt | 4 +- src/tool/cmptest/src/testutil.rs | 11 +--- 4 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 spec/brgen_cmptest_config_schema.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 616e8e6f..46a08ea4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,6 +70,7 @@ jobs: PREFIX=$(brew --prefix llvm) # from https://github.com/llvm/llvm-project/issues/77786 LIBCPP=$PREFIX/lib/c++ + ls $LIBCPP nm $LIBCPP/libc++.1.dylib | grep typeinfo - name: Build run: | diff --git a/spec/brgen_cmptest_config_schema.json b/spec/brgen_cmptest_config_schema.json new file mode 100644 index 00000000..250075d5 --- /dev/null +++ b/spec/brgen_cmptest_config_schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "description": "brgen(BinaRy encoder/decoder GENerator driver) cmptest utility configuration schema", + "properties": { + "inputs": { + "type": "array", + "description": "List of input files to be used for testing", + "items": { + "type": "object", + "properties": { + "binary": { + "type": "string", + "description": "binary data file to be used for testing" + }, + "format_name": { + "type": "string", + "description": "format name of test target" + }, + "file_base": { + "type": "string", + "description": "base name of test target file" + }, + "failure_case": { + "type": "boolean", + "description": "flag to indicate that the test input binary of this file is failure case" + } + } + } + }, + "runners": { + "type": "array", + "description": "List of runner configurations", + "items": { + "type": "object", + "properties": { + "suffix": { + "type": "string", + "description": "suffix of target generated file" + }, + "test_template": { + "type": "string", + "description": "template file for test runner" + }, + "replace_struct_name": { + "type": "string", + "description": "struct name to be replaced in test runner template" + }, + "replace_file_name": { + "type": "string", + "description": "file name to be replaced in test runner template" + }, + "build_input_name": { + "type": "string", + "description": "input file name to be used for building test runner" + }, + "build_output_name": { + "type": "string", + "description": "output file name to be generated by building test runner" + }, + "build_command": { + "type": "array", + "description": "command to build test runner. $INPUT and $OUTPUT will be replaced with build_input_name and build_output_name", + "items": { + "type": "string" + } + }, + "run_command": { + "type": "array", + "description": "command to run test runner.$EXEC is replaced with built executable (or script) by build_command and $INPUT and $OUTPUT will be replaced with input binary and output binary path" + } + } + } + } + }, + "additionalProperties": false +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c5d887e9..77a0bc48 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -303,6 +303,6 @@ if("$ENV{BRGEN_RUST_ENABLED}" STREQUAL "1") else() set(RUST_TARGET_DIR "release") endif() - install(PROGRAMS "${CMAKE_SOURCE_DIR}/src/tool/json2rust/target/${RUST_TARGET_DIR}/json2rust${SUFFIX}" DESTINATION tool OPTIONAL) - install(PROGRAMS "${CMAKE_SOURCE_DIR}/src/tool/cmptest/target/${RUST_TARGET_DIR}/cmptest${SUFFIX}" DESTINATION tool OPTIONAL) + install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/json2rust/target/${RUST_TARGET_DIR}/json2rust${SUFFIX}" DESTINATION tool OPTIONAL) + install(PROGRAMS "${CMAKE_BINARY_DIR}/tool/cmptest/target/${RUST_TARGET_DIR}/cmptest${SUFFIX}" DESTINATION tool OPTIONAL) endif() diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index f5c7a5c3..9cbbf370 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -1,12 +1,4 @@ - -use std::{ - collections::HashMap, - env, - ffi::OsStr, - fs, os, - path::{Path, PathBuf}, - process, -}; +use std::{collections::HashMap, env, fs, path::PathBuf, process}; use rand::{ self, @@ -108,7 +100,6 @@ pub struct TestSchedule<'a> { pub file: &'a GeneratedFileInfo, } - pub type Error = Box; pub struct TestScheduler { From c06054a55e5bf4307d2f296a6268fe20ab4846d1 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Fri, 26 Apr 2024 23:25:39 +0900 Subject: [PATCH 24/50] add cmptest.json --- src/tool/cmptest/src/testutil.rs | 36 ++++++++++----- testkit/cmptest.json | 27 +++++++++++ testkit/{ => cpp}/CMakeLists.txt | 4 +- testkit/cpp/setup.py | 35 +++++++++++++++ testkit/{ => cpp}/stub.hpp | 0 testkit/cpp/test_template.cpp | 77 ++++++++++++++++++++++++++++++++ testkit/test_template.cpp | 62 ------------------------- 7 files changed, 167 insertions(+), 74 deletions(-) create mode 100644 testkit/cmptest.json rename testkit/{ => cpp}/CMakeLists.txt (55%) create mode 100644 testkit/cpp/setup.py rename testkit/{ => cpp}/stub.hpp (100%) create mode 100644 testkit/cpp/test_template.cpp delete mode 100644 testkit/test_template.cpp diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index 9cbbf370..56be727b 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -117,7 +117,7 @@ impl TestScheduler { } } - fn read_template(&mut self, path: &str) -> Result { + fn read_text_file(&mut self, path: &str) -> Result { if let Some(x) = self.template_files.get(path) { return Ok(x.clone()); } else { @@ -152,7 +152,7 @@ impl TestScheduler { fn prepare_content<'a>(&mut self, sched: &TestSchedule<'a>) -> Result { // get template and replace with target - let template = self.read_template(&sched.runner.test_template)?; + let template = self.read_text_file(&sched.runner.test_template)?; let replace_with = &sched.runner.replace_struct_name; let template = template.replace(replace_with, &sched.input.format_name); let replace_with = &sched.runner.replace_file_name; @@ -161,24 +161,33 @@ impl TestScheduler { Ok(instance) } - fn create_input_file<'a>( + fn create_test_dir<'a>( &mut self, - sched: &TestSchedule<'a>, - instance: String, - ) -> Result<(PathBuf, PathBuf, PathBuf), Error> { + sched: &TestSchedule<'a> + ) -> Result { let tmp_dir = self.get_tmp_dir(); let tmp_dir = tmp_dir.join(&sched.file.base); let tmp_dir = tmp_dir.join(&sched.input.format_name); let tmp_dir = tmp_dir.join(&sched.file.suffix); fs::create_dir_all(&tmp_dir)?; + Ok(tmp_dir) + } + + fn create_input_file<'a>( + &mut self, + sched: &TestSchedule<'a>, + tmp_dir: &PathBuf, + instance: String, + ) -> Result<( PathBuf, PathBuf), Error> { let input_file = tmp_dir.join(&sched.runner.build_input_name); let output_file = tmp_dir.join(&sched.runner.build_output_name); fs::write(&input_file, instance)?; - Ok((tmp_dir, input_file, output_file)) + Ok((input_file, output_file)) } - fn replace_cmd( + fn replace_cmd<'a>( cmd: &mut Vec, + sched : &TestSchedule<'a>, tmp_dir: &PathBuf, input: &PathBuf, output: &PathBuf, @@ -199,11 +208,15 @@ impl TestScheduler { if c == "$TMPDIR" { *c = tmp_dir.to_str().unwrap().to_string(); } + if c == "$ORIGIN" { + *c = format!("{}/{}", sched.file.dir, sched.file.base); + } } } fn exec_cmd<'a>( &mut self, + sched :&TestSchedule<'a>, base: &Vec, tmp_dir: &PathBuf, input: &PathBuf, @@ -212,7 +225,7 @@ impl TestScheduler { expect_ok: bool, ) -> Result { let mut cmd = base.clone(); - Self::replace_cmd(&mut cmd, tmp_dir, input, output, exec); + Self::replace_cmd(&mut cmd,sched, tmp_dir, input, output, exec); let mut r = process::Command::new(&cmd[0]); r.args(&cmd[1..]); let done = r.output()?; @@ -236,12 +249,14 @@ impl TestScheduler { } pub fn run_test_schedule<'a>(&mut self, sched: &TestSchedule<'a>) -> Result<(), Error> { + let tmp_dir = self.create_test_dir(sched)?; let instance = self.prepare_content(sched)?; - let (tmp_dir, input, output) = self.create_input_file(sched, instance)?; + let ( input, output) = self.create_input_file(sched, &tmp_dir,instance)?; // build test self.exec_cmd( + &sched, &sched.runner.build_command, &tmp_dir, &input, @@ -258,6 +273,7 @@ impl TestScheduler { // run test let status = self.exec_cmd( + &sched, &sched.runner.run_command, &tmp_dir, &input_binary, diff --git a/testkit/cmptest.json b/testkit/cmptest.json new file mode 100644 index 00000000..d9f98682 --- /dev/null +++ b/testkit/cmptest.json @@ -0,0 +1,27 @@ +{ + "inputs": [ + { + "binary": "./example/wire_data/websocket.dat", + "format_name": "Frame", + "file_base": "websocket", + "failure_case": false + } + ], + "runner": [ + { + "suffix": ".hpp", + "test_template": "./testkit/cpp/test_template.cpp", + "replace_file_name": "stub.hpp", + "replace_struct_name": "TEST_CLASS", + "build_input_name": "test.cpp", + "build_output_name": "test.exe", + "build_command": [ + "python", + "./testkit/cpp/setup.py", + "$INPUT", + "$OUTPUT" + ], + "run_command": ["$EXEC", "$INPUT", "$OUTPUT"] + } + ] +} diff --git a/testkit/CMakeLists.txt b/testkit/cpp/CMakeLists.txt similarity index 55% rename from testkit/CMakeLists.txt rename to testkit/cpp/CMakeLists.txt index 60d2c459..e44635ff 100644 --- a/testkit/CMakeLists.txt +++ b/testkit/cpp/CMakeLists.txt @@ -5,6 +5,6 @@ project(GeneratedCodeTest CXX) set(CMAKE_CXX_STANDARD 20) -include_directories("$ENV{FUTILS_DIR}/src/include") -link_directories("$ENV{FUTILS_DIR}/lib") +include_directories("${FUTILS_DIR}/src/include") +link_directories("${FUTILS_DIR}/lib") diff --git a/testkit/cpp/setup.py b/testkit/cpp/setup.py new file mode 100644 index 00000000..edb93280 --- /dev/null +++ b/testkit/cpp/setup.py @@ -0,0 +1,35 @@ +import sys + +INPUT = sys.argv[1] +OUTPUT = sys.argv[2] + +import os + +FUTILS_DIR = os.environ.get("FUTILS_DIR") +if FUTILS_DIR is None: + print("FUTILS_DIR not set", file=sys.stderr) + exit(2) + +import subprocess as sp + +# run compiler for the test +# add link directory $FUTILS_DIR/lib and -lfutils +# compiler output will be redirected to stdout and stderr +CC = "clang++" + +sp.call( + [ + CC, + "-std=c++20", + "-I", + FUTILS_DIR + "/include", + "-L", + FUTILS_DIR + "/lib", + "-lfutils", + INPUT, + "-o", + OUTPUT, + ], + stdout=sys.stdout, + stderr=sys.stderr, +) diff --git a/testkit/stub.hpp b/testkit/cpp/stub.hpp similarity index 100% rename from testkit/stub.hpp rename to testkit/cpp/stub.hpp diff --git a/testkit/cpp/test_template.cpp b/testkit/cpp/test_template.cpp new file mode 100644 index 00000000..c5b604d0 --- /dev/null +++ b/testkit/cpp/test_template.cpp @@ -0,0 +1,77 @@ +// Created by make_cpp_test.sh +#include "stub.hpp" +#include +#include +#include +#include +#include +auto& cout = futils::wrap::cerr_wrap(); + +int do_encode(char** argv, auto& t, auto& w) { + if constexpr (std::is_same_v) { + if (!t.encode(w)) { + cout << "Failed to encode file " << argv[1] << "; encode() returns false\n"; + return 2; + } + } + else { + if (auto err = t.encode(w)) { + cout << "Failed to encode file " << argv[1] << "; encode() returns " << err.template error() << '\n'; + return 2; + } + } + return 0; +} + +int do_decode(char** argv, auto& t, auto& r) { + if constexpr (std::is_same_v) { + if (!t.decode(r)) { + cout << "Failed to decode file " << argv[1] << "; decode() returns false\n"; + return 1; + } + } + else { + if (auto err = t.decode(r)) { + cout << "Failed to decode file " << argv[1] << "; decode() returns " << err.template error() << '\n'; + return 1; + } + } + return 0; +} + +int main(int argc, char** argv) { + if (argc < 3) { + cout << "Usage: " << argv[0] << " \n"; + return 2; + } + TEST_CLASS t; + futils::file::View view; + if (!view.open(argv[1])) { + cout << "Failed to open file " << argv[1] << '\n'; + return 2; + } + if (!view.data()) { + cout << "Failed to read file " << argv[1] << '\n'; + return 2; + } + futils::binary::reader r{view}; + if (auto e = do_decode(argv, t, r)) { + return e; + } + if (!r.empty()) { + cout << "Failed to decode file " << argv[1] << ";" << r.remain().size() << " bytes left\n"; + return 1; + } + std::string s; + s.resize(view.size()); + futils::binary::writer w{futils::binary::resizable_buffer_writer(), &s}; + if (auto e = do_encode(argv, t, w)) { + return e; + } + std::ofstream fs(argv[2], std::ios::binary | std::ios::out); + if (!fs) { + cout << "Failed to open file " << argv[2] << '\n'; + return 2; + } + fs << s; +} diff --git a/testkit/test_template.cpp b/testkit/test_template.cpp deleted file mode 100644 index 69a97192..00000000 --- a/testkit/test_template.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// Created by make_cpp_test.sh -#include "stub.hpp" -#include -#include -#include -#include - -int main(int argc, char** argv) { - auto& cout = futils::wrap::cout_wrap(); - if (argc < 2) { - cout << "Usage: " << argv[0] << " \n"; - return 1; - } - TEST_CLASS t; - futils::file::View view; - if (!view.open(argv[1])) { - cout << "Failed to open file " << argv[1] << '\n'; - return 1; - } - if (!view.data()) { - cout << "Failed to read file " << argv[1] << '\n'; - return 1; - } - futils::binary::reader r{view}; - if (!t.decode(r)) { - cout << "Failed to decode file " << argv[1] << "; decode() returns false\n"; - return 1; - } - if (!r.empty()) { - cout << "Failed to decode file " << argv[1] << ";" << r.remain().size() << " bytes left\n"; - return 1; - } - std::string s; - s.resize(view.size()); - futils::binary::writer w{s}; - if (!t.encode(w)) { - cout << "Failed to encode file " << argv[1] << "; encode() returns false\n"; - return 1; - } - if (!w.full()) { - cout << "Failed to encode file " << argv[1] << ";" << w.remain().size() << " bytes left\n"; - return 1; - } - if (futils::view::rvec(view) != s) { - cout << "Failed to encode file " << argv[1] << "; encoded data is different\n"; - return 1; - } - view.close(); - for (auto i = 0; i < 10000; i++) { - futils::binary::reader r{s}; - if (!t.decode(r)) { - cout << "Failed to decode repeat test " << argv[1] << "; decode() returns false\n"; - return 1; - } - futils::binary::writer w{s}; - if (!t.encode(w)) { - cout << "Failed to encode repeat test " << argv[1] << "; encode() returns false\n"; - return 1; - } - } - return 0; -} From 824225ef8f1450426bd14517d72bd24959814a0e Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 01:18:01 +0900 Subject: [PATCH 25/50] fixing test --- .github/workflows/build.yml | 2 +- spec/brgen_cmptest_config_schema.json | 4 ++ src/tool/cmptest/src/main.rs | 50 +++++++++++--- src/tool/cmptest/src/testutil.rs | 96 ++++++++++++++++++++------- testkit/cmptest.json | 5 +- 5 files changed, 121 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 46a08ea4..b69c06e2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,7 +71,7 @@ jobs: # from https://github.com/llvm/llvm-project/issues/77786 LIBCPP=$PREFIX/lib/c++ ls $LIBCPP - nm $LIBCPP/libc++.1.dylib | grep typeinfo + nm $LIBCPP/libc++.1.dylib - name: Build run: | PREFIX=$(brew --prefix llvm) diff --git a/spec/brgen_cmptest_config_schema.json b/spec/brgen_cmptest_config_schema.json index 250075d5..5168c1f9 100644 --- a/spec/brgen_cmptest_config_schema.json +++ b/spec/brgen_cmptest_config_schema.json @@ -24,6 +24,10 @@ "failure_case": { "type": "boolean", "description": "flag to indicate that the test input binary of this file is failure case" + }, + "hex_input": { + "type": "boolean", + "description": "flag to indicate that the test input is hex string" } } } diff --git a/src/tool/cmptest/src/main.rs b/src/tool/cmptest/src/main.rs index 4d7ab9b1..a5a433e3 100644 --- a/src/tool/cmptest/src/main.rs +++ b/src/tool/cmptest/src/main.rs @@ -11,10 +11,12 @@ struct Args { test_info_file: String, #[arg(long, short('c'))] test_config_file: String, + #[arg(long, short('d'))] + debug: bool, } fn main() -> Result<(), Error> { let parsed = Args::parse(); - let test_config =fs::read_to_string(Path::new (&parsed.test_config_file))?; + let test_config = fs::read_to_string(Path::new(&parsed.test_config_file))?; let mut d1 = serde_json::Deserializer::from_str(&test_config); let test_config = testutil::TestConfig::deserialize(&mut d1)?; let mut ext_set = std::collections::HashMap::new(); @@ -58,6 +60,9 @@ fn main() -> Result<(), Error> { ); continue; } + if parsed.debug { + eprint!("runner: {:?}", runner) + } ext_set.insert(runner.suffix.clone(), runner.clone()); } for input in &test_config.inputs { @@ -71,6 +76,9 @@ fn main() -> Result<(), Error> { let mut d2 = serde_json::Deserializer::from_str(&test_info); let test_info = testutil::TestInfo::deserialize(&mut d2)?; for file in &test_info.generated_files { + if parsed.debug { + eprintln!("file: {:?}", file); + } if file.suffix != ".json" { continue; } @@ -82,7 +90,12 @@ fn main() -> Result<(), Error> { let ext = split[1]; let runner = match ext_set.get(ext) { Some(x) => x, - None => continue, + None => { + if parsed.debug { + eprintln!("runner not found for ext: {}", ext); + } + continue; + } }; let path = file.into_path(); let content = fs::read_to_string(&path)?; @@ -95,7 +108,14 @@ fn main() -> Result<(), Error> { } }; for s in &data.structs { - if let Some(input) = file_format_list.get(&(base.to_string(), s.clone())) { + let key = (base.to_string(), s.clone()); + if parsed.debug { + eprintln!("key: {:?}", key); + } + if let Some(input) = file_format_list.get(&key) { + if parsed.debug { + eprintln!("input matched: {:?}", input); + } sched.push(TestSchedule { runner: runner, input: input, @@ -105,18 +125,28 @@ fn main() -> Result<(), Error> { } } let mut scheduler = testutil::TestScheduler::new(); - for s in sched { + let mut failed = 0; + for s in &sched { match scheduler.run_test_schedule(&s) { Ok(()) => println!( "test passed: {:?}", - s.file.into_path() + &s.input.format_name - ), - Err(x) => println!( - "test failed: {:?} {:?}", - s.file.into_path() + &s.input.format_name, - x + s.file.into_path() + "/" + &s.input.format_name ), + Err(x) => { + println!( + "test failed: {:?} {:?}", + s.file.into_path() + "/" + &s.input.format_name, + x + ); + failed += 1; + } } } + println!( + "test finished: total/pass/failed = {}/{}/{}", + sched.len(), + sched.len() - failed, + failed + ); Ok(()) } diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index 56be727b..71e2e937 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -8,7 +8,7 @@ use rand::{ use ast2rust::ast; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize,Debug)] pub struct LineMap { pub line: u64, @@ -16,7 +16,7 @@ pub struct LineMap { pub loc: ast::Loc, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize,Debug)] pub struct GeneratedData { pub structs: Vec, @@ -24,7 +24,7 @@ pub struct GeneratedData { pub line_map: Vec, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize,Debug)] pub struct GeneratedFileInfo { pub dir: String, @@ -34,11 +34,11 @@ pub struct GeneratedFileInfo { pub suffix: String, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize ,Debug)] pub struct TestInfo { pub total_count: u64, - pub err_count: u64, + pub error_count: u64, pub time: String, @@ -51,7 +51,7 @@ impl GeneratedFileInfo { } } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone,Debug)] pub struct TestRunner { // file suffix of generated files pub suffix: String, @@ -75,7 +75,7 @@ pub struct TestRunner { pub run_command: Vec, } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone,Debug)] pub struct TestInput { // input binary file pub binary: String, @@ -85,6 +85,8 @@ pub struct TestInput { pub file_base: String, // this input is failure case pub failure_case: bool, + // binary is actually hex string file + pub hex: bool, } #[derive(Serialize, Deserialize, Clone)] @@ -105,7 +107,7 @@ pub type Error = Box; pub struct TestScheduler { template_files: HashMap, tmpdir: Option, - input_binaries: HashMap>, + input_binaries: HashMap>, } impl TestScheduler { @@ -127,12 +129,62 @@ impl TestScheduler { } } - fn read_input_binary(&mut self, path: &str) -> Result, Error> { + fn compile_hex(input: Vec) -> Result, Error> { + // skip space,tab,line and # line comment + let mut hex = Vec::new(); + let mut i = 0; + let mut pair: Option = None; + while i < input.len() { + let c = input[i]; + if c == b' ' || c == b'\t' || c == b'\n' { + i += 1; + continue; + } + if c == b'#' { + while i < input.len() && input[i] != b'\n' { + i += 1; + } + continue; + } + let lsb = if b'0' <= c && c <= b'9' { + c - b'0' + } else if b'a' <= c && c <= b'f' { + c - b'a' + 10 + } else if b'A' <= c && c <= b'F' { + c - b'A' + 10 + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("invalid hex string at {}:{}", i, c), + ) + .into()); + }; + if let Some(msb) = pair { + hex.push(msb << 4 | lsb); + pair = None + } else { + pair = Some(lsb) + } + i += 1; + } + if let Some(_) = pair { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "invalid hex string; missing pair", + ) + .into()) + } else { + Ok(hex) + } + } + + fn read_input_binary(&mut self, path: &PathBuf, is_hex: bool) -> Result, Error> { if let Some(x) = self.input_binaries.get(path) { return Ok(x.clone()); } else { let t = fs::read(path)?; - self.input_binaries.insert(path.to_string(), t); + let t = if is_hex { Self::compile_hex(t)? } else { t }; + self.input_binaries.insert(path.clone(), t); Ok(self.input_binaries.get(path).unwrap().clone()) } } @@ -161,10 +213,7 @@ impl TestScheduler { Ok(instance) } - fn create_test_dir<'a>( - &mut self, - sched: &TestSchedule<'a> - ) -> Result { + fn create_test_dir<'a>(&mut self, sched: &TestSchedule<'a>) -> Result { let tmp_dir = self.get_tmp_dir(); let tmp_dir = tmp_dir.join(&sched.file.base); let tmp_dir = tmp_dir.join(&sched.input.format_name); @@ -178,7 +227,7 @@ impl TestScheduler { sched: &TestSchedule<'a>, tmp_dir: &PathBuf, instance: String, - ) -> Result<( PathBuf, PathBuf), Error> { + ) -> Result<(PathBuf, PathBuf), Error> { let input_file = tmp_dir.join(&sched.runner.build_input_name); let output_file = tmp_dir.join(&sched.runner.build_output_name); fs::write(&input_file, instance)?; @@ -187,7 +236,7 @@ impl TestScheduler { fn replace_cmd<'a>( cmd: &mut Vec, - sched : &TestSchedule<'a>, + sched: &TestSchedule<'a>, tmp_dir: &PathBuf, input: &PathBuf, output: &PathBuf, @@ -216,7 +265,7 @@ impl TestScheduler { fn exec_cmd<'a>( &mut self, - sched :&TestSchedule<'a>, + sched: &TestSchedule<'a>, base: &Vec, tmp_dir: &PathBuf, input: &PathBuf, @@ -225,7 +274,7 @@ impl TestScheduler { expect_ok: bool, ) -> Result { let mut cmd = base.clone(); - Self::replace_cmd(&mut cmd,sched, tmp_dir, input, output, exec); + Self::replace_cmd(&mut cmd, sched, tmp_dir, input, output, exec); let mut r = process::Command::new(&cmd[0]); r.args(&cmd[1..]); let done = r.output()?; @@ -252,7 +301,7 @@ impl TestScheduler { let tmp_dir = self.create_test_dir(sched)?; let instance = self.prepare_content(sched)?; - let ( input, output) = self.create_input_file(sched, &tmp_dir,instance)?; + let (input, output) = self.create_input_file(sched, &tmp_dir, instance)?; // build test self.exec_cmd( @@ -269,14 +318,16 @@ impl TestScheduler { let output = tmp_dir.join("output.bin"); - let input_binary = sched.input.binary.clone().into(); + let input_binary_name: PathBuf = sched.input.binary.clone().into(); + + let input_binary = self.read_input_binary(&input_binary_name, sched.input.hex)?; // check input is valid // run test let status = self.exec_cmd( &sched, &sched.runner.run_command, &tmp_dir, - &input_binary, + &input_binary_name, &output, Some(&exec), false, @@ -292,7 +343,6 @@ impl TestScheduler { .into()); } - let input_binary = self.read_input_binary(&input_binary.to_string_lossy())?; // check input is valid let output = fs::read(&output)?; // check output is valid let min_size = if input_binary.len() < output.len() { @@ -322,7 +372,7 @@ impl TestScheduler { } if !diff.is_empty() { - let mut debug = format!("test failed: input and output is different\n"); + let mut debug = format!("input and output is different\n"); for (i, a, b) in diff { debug += &format!("{}: {:02x?} != {:02x?}\n", i, a, b); } diff --git a/testkit/cmptest.json b/testkit/cmptest.json index d9f98682..814f70ac 100644 --- a/testkit/cmptest.json +++ b/testkit/cmptest.json @@ -4,10 +4,11 @@ "binary": "./example/wire_data/websocket.dat", "format_name": "Frame", "file_base": "websocket", - "failure_case": false + "failure_case": false, + "hex": true } ], - "runner": [ + "runners": [ { "suffix": ".hpp", "test_template": "./testkit/cpp/test_template.cpp", From a64dfa291b13f41b54b9b1f3213d320763b55ee0 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 01:20:47 +0900 Subject: [PATCH 26/50] c++filt --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b69c06e2..29c94e93 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,7 +71,7 @@ jobs: # from https://github.com/llvm/llvm-project/issues/77786 LIBCPP=$PREFIX/lib/c++ ls $LIBCPP - nm $LIBCPP/libc++.1.dylib + nm $LIBCPP/libc++.1.dylib | c++filt - name: Build run: | PREFIX=$(brew --prefix llvm) From 0fc37155ab99916cf23c5b024aa7709f58036a5c Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 01:25:16 +0900 Subject: [PATCH 27/50] testing --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 29c94e93..4265b42d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,6 +72,7 @@ jobs: LIBCPP=$PREFIX/lib/c++ ls $LIBCPP nm $LIBCPP/libc++.1.dylib | c++filt + nm $LIBCPP/libc++abi.1.dylib | c++filt - name: Build run: | PREFIX=$(brew --prefix llvm) From be214b62d3fc1251606c806c1342cd2950bc694a Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 01:39:26 +0900 Subject: [PATCH 28/50] echo --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4265b42d..4b34ec2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,7 +71,9 @@ jobs: # from https://github.com/llvm/llvm-project/issues/77786 LIBCPP=$PREFIX/lib/c++ ls $LIBCPP + echo "libc++" nm $LIBCPP/libc++.1.dylib | c++filt + echo "libc++abi" nm $LIBCPP/libc++abi.1.dylib | c++filt - name: Build run: | From edbe94b2451fb8614e5bde4bb1ace09fe6e9b9ff Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 02:12:05 +0900 Subject: [PATCH 29/50] fix --- src/tool/cmptest/src/main.rs | 8 +++++++- src/tool/cmptest/src/testutil.rs | 14 +++++++------- testkit/cmptest.json | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/tool/cmptest/src/main.rs b/src/tool/cmptest/src/main.rs index a5a433e3..7e834d79 100644 --- a/src/tool/cmptest/src/main.rs +++ b/src/tool/cmptest/src/main.rs @@ -98,7 +98,13 @@ fn main() -> Result<(), Error> { } }; let path = file.into_path(); - let content = fs::read_to_string(&path)?; + let content = match fs::read_to_string(&path) { + Ok(x) => x, + Err(x) => { + eprintln!("file load error: {}: {}", path, x); + continue + } + }; let mut d = serde_json::Deserializer::from_str(&content); let data = match testutil::GeneratedData::deserialize(&mut d) { Ok(file) => file, diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index 71e2e937..0a7bfb32 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -8,7 +8,7 @@ use rand::{ use ast2rust::ast; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize,Debug)] +#[derive(Serialize, Deserialize, Debug)] pub struct LineMap { pub line: u64, @@ -16,7 +16,7 @@ pub struct LineMap { pub loc: ast::Loc, } -#[derive(Serialize, Deserialize,Debug)] +#[derive(Serialize, Deserialize, Debug)] pub struct GeneratedData { pub structs: Vec, @@ -24,7 +24,7 @@ pub struct GeneratedData { pub line_map: Vec, } -#[derive(Serialize, Deserialize,Debug)] +#[derive(Serialize, Deserialize, Debug)] pub struct GeneratedFileInfo { pub dir: String, @@ -34,7 +34,7 @@ pub struct GeneratedFileInfo { pub suffix: String, } -#[derive(Serialize, Deserialize ,Debug)] +#[derive(Serialize, Deserialize, Debug)] pub struct TestInfo { pub total_count: u64, @@ -47,11 +47,11 @@ pub struct TestInfo { impl GeneratedFileInfo { pub fn into_path(&self) -> String { - format!("{}/{}.{}", self.dir, self.base, self.suffix) + format!("{}/{}{}", self.dir, self.base, self.suffix) } } -#[derive(Serialize, Deserialize, Clone,Debug)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct TestRunner { // file suffix of generated files pub suffix: String, @@ -75,7 +75,7 @@ pub struct TestRunner { pub run_command: Vec, } -#[derive(Serialize, Deserialize, Clone,Debug)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct TestInput { // input binary file pub binary: String, diff --git a/testkit/cmptest.json b/testkit/cmptest.json index 814f70ac..066181c1 100644 --- a/testkit/cmptest.json +++ b/testkit/cmptest.json @@ -10,7 +10,7 @@ ], "runners": [ { - "suffix": ".hpp", + "suffix": "hpp", "test_template": "./testkit/cpp/test_template.cpp", "replace_file_name": "stub.hpp", "replace_struct_name": "TEST_CLASS", From 00fc5f375e85ab1741b632399335c4e2988cde00 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 02:22:41 +0900 Subject: [PATCH 30/50] fix --- src/tool/cmptest/src/testutil.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index 0a7bfb32..f98a1ae8 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -134,9 +134,16 @@ impl TestScheduler { let mut hex = Vec::new(); let mut i = 0; let mut pair: Option = None; + let mut line = 1; + let mut col = 1; while i < input.len() { let c = input[i]; - if c == b' ' || c == b'\t' || c == b'\n' { + if c == b' ' || c == b'\t' || c == b'\n'||c==b'\r' { + if c==b'\n' { + line+=1; + col=0; + } + col+=1; i += 1; continue; } @@ -155,7 +162,7 @@ impl TestScheduler { } else { return Err(std::io::Error::new( std::io::ErrorKind::Other, - format!("invalid hex string at {}:{}", i, c), + format!("invalid hex string at {}:{}:{}", line,col, c as char), ) .into()); }; @@ -166,6 +173,7 @@ impl TestScheduler { pair = Some(lsb) } i += 1; + col+=1; } if let Some(_) = pair { Err(std::io::Error::new( From 45c328a1c83fa965f0e0582b1ce9ac85eeae23ea Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 02:39:33 +0900 Subject: [PATCH 31/50] fix! --- src/tool/cmptest/src/testutil.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index f98a1ae8..9fdfae92 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -123,7 +123,13 @@ impl TestScheduler { if let Some(x) = self.template_files.get(path) { return Ok(x.clone()); } else { - let t = fs::read_to_string(path)?; + let t = match fs::read_to_string(path) { + Ok(x) => x, + Err(x)=> { + return Err(std::io::Error::new(std::io::ErrorKind::Other, + format!("read file error: {}: {}",path,x)).into()); + } + }; self.template_files.insert(path.to_string(), t); Ok(self.template_files.get(path).unwrap().clone()) } @@ -190,7 +196,13 @@ impl TestScheduler { if let Some(x) = self.input_binaries.get(path) { return Ok(x.clone()); } else { - let t = fs::read(path)?; + let t = match fs::read(path) { + Ok(x) => x, + Err(x)=> { + return Err(std::io::Error::new(std::io::ErrorKind::Other, + format!("read file error: {}: {}",path,x)).into()); + } + }; let t = if is_hex { Self::compile_hex(t)? } else { t }; self.input_binaries.insert(path.clone(), t); Ok(self.input_binaries.get(path).unwrap().clone()) From 47afa933d6c61fc242ed06fe91110285301bb113 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sat, 27 Apr 2024 02:43:50 +0900 Subject: [PATCH 32/50] fix! --- src/tool/cmptest/src/testutil.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index 9fdfae92..bc24b0c4 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -200,7 +200,7 @@ impl TestScheduler { Ok(x) => x, Err(x)=> { return Err(std::io::Error::new(std::io::ErrorKind::Other, - format!("read file error: {}: {}",path,x)).into()); + format!("read file error: {:?}: {}",path,x)).into()); } }; let t = if is_hex { Self::compile_hex(t)? } else { t }; From f5a2b6f1a93409ed7f7f62d5ccf1abb1b600c487 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sun, 28 Apr 2024 16:13:30 +0900 Subject: [PATCH 33/50] generate config --- example/webauthn.bgn | 43 +++ spec/brgen_cmptest_config_schema.json | 4 + src/tool/cmptest/Cargo.lock | 324 +++++++++++++++++++++- src/tool/cmptest/Cargo.toml | 2 + src/tool/cmptest/src/main.rs | 113 +++++--- src/tool/cmptest/src/testutil.rs | 380 ++++++++++++++++++-------- src/tool/json2ts/generate.cpp | 239 ++++++++++++---- testkit/cmptest.json | 21 +- testkit/cpp/config.json | 17 ++ testkit/cpp/setup.py | 58 ++-- testkit/cpp/test_template.cpp | 2 +- 11 files changed, 963 insertions(+), 240 deletions(-) create mode 100644 example/webauthn.bgn create mode 100644 testkit/cpp/config.json diff --git a/example/webauthn.bgn b/example/webauthn.bgn new file mode 100644 index 00000000..3eea634b --- /dev/null +++ b/example/webauthn.bgn @@ -0,0 +1,43 @@ + +format PublicKeyCredential: + raw_id_len :u8 + raw_id :[raw_id_len]u8 + auth_attestation_response :AuthenticatorAttestationResponse + +format AuthenticatorResponse: + client_data_json_len :u16 + client_data_json :[client_data_json_len]u8 + type :AuthenticatorResponseType + if type == AuthenticatorResponseType.attestation: + attestation :AuthenticatorAttestationResponse + else: + assertion :AuthenticatorAssertionResponse + +enum AuthenticatorResponseType: + :u8 + attestation = 0 + assertion = 1 + +format AuthenticatorAttestationResponse: + attestation_len :u16 + attestation_response :[attestation_len]u8 + + +format AuthenticatorAssertionResponse: + authenticatorData_len :u16 + authenticatorData :[authenticatorData_len]u8 + signature_len :u16 + signature :[signature_len]u8 + user_handle_len :u16 + user_handle :[user_handle_len]u8 + +enum Algorithm: + :s8 + Ed25519 = -8 + ES256 = -7 + RS256 = -257 + +format CreateInfo: + challenge_len :u8 + challenge :[challenge_len]u8 + \ No newline at end of file diff --git a/spec/brgen_cmptest_config_schema.json b/spec/brgen_cmptest_config_schema.json index 5168c1f9..608cf2ed 100644 --- a/spec/brgen_cmptest_config_schema.json +++ b/spec/brgen_cmptest_config_schema.json @@ -38,6 +38,10 @@ "items": { "type": "object", "properties": { + "file": { + "type": "string", + "description": "file name of runner configuration that contains an JSON object with below properties" + }, "suffix": { "type": "string", "description": "suffix of target generated file" diff --git a/src/tool/cmptest/Cargo.lock b/src/tool/cmptest/Cargo.lock index 8ff31353..a2a59ccf 100644 --- a/src/tool/cmptest/Cargo.lock +++ b/src/tool/cmptest/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -46,7 +61,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -56,7 +71,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -68,6 +83,45 @@ dependencies = [ "serde_json", ] +[[package]] +name = "autocfg" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" + +[[package]] +name = "backtrace" +version = "0.3.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cc" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" + [[package]] name = "cfg-if" version = "1.0.0" @@ -120,10 +174,12 @@ version = "0.1.0" dependencies = [ "ast2rust", "clap", + "colored", "rand", "regex", "serde", "serde_json", + "tokio", ] [[package]] @@ -132,6 +188,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "getrandom" version = "0.2.14" @@ -143,30 +209,126 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "memchr" version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "parking_lot" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.5", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -221,6 +383,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.10.4" @@ -250,12 +421,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "ryu" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.198" @@ -287,6 +470,31 @@ dependencies = [ "serde", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "strsim" version = "0.11.1" @@ -304,6 +512,36 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tokio" +version = "1.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.12" @@ -322,13 +560,37 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -337,28 +599,46 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.5" @@ -371,24 +651,48 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.5" diff --git a/src/tool/cmptest/Cargo.toml b/src/tool/cmptest/Cargo.toml index f5182ad5..9af10a2f 100644 --- a/src/tool/cmptest/Cargo.toml +++ b/src/tool/cmptest/Cargo.toml @@ -12,3 +12,5 @@ serde_json = "1.0.116" ast2rust = {path ="../../../ast2rust" } regex = "1.10.4" rand = "0.8.5" +tokio = { version = "1.37.0", features = ["full"] } +colored = "2.1.0" diff --git a/src/tool/cmptest/src/main.rs b/src/tool/cmptest/src/main.rs index 7e834d79..5ba4837e 100644 --- a/src/tool/cmptest/src/main.rs +++ b/src/tool/cmptest/src/main.rs @@ -1,9 +1,10 @@ -use std::{fs, path::Path}; +use std::{fs, path::Path, sync::Arc}; use clap::{arg, Parser}; use serde::Deserialize; use testutil::{Error, TestSchedule}; mod testutil; +use colored::Colorize; #[derive(Parser)] struct Args { @@ -13,15 +14,33 @@ struct Args { test_config_file: String, #[arg(long, short('d'))] debug: bool, + #[arg(long)] + save_tmp_dir: bool, } -fn main() -> Result<(), Error> { + +#[tokio::main] +async fn main() -> Result<(), Error> { let parsed = Args::parse(); let test_config = fs::read_to_string(Path::new(&parsed.test_config_file))?; let mut d1 = serde_json::Deserializer::from_str(&test_config); - let test_config = testutil::TestConfig::deserialize(&mut d1)?; + let mut test_config = testutil::TestConfig::deserialize(&mut d1)?; let mut ext_set = std::collections::HashMap::new(); - let mut file_format_list = std::collections::HashMap::new(); - for runner in &test_config.runners { + let mut file_format_list:std::collections::HashMap<(String,String),Vec>> = std::collections::HashMap::new(); + for runner_config in &mut test_config.runners { + let runner = match runner_config { + testutil::RunnerConfig::TestRunner(x) => x, + testutil::RunnerConfig::File(x) => { + let r = fs::read(&x.file)?; + let mut de = serde_json::Deserializer::from_slice(&r); + let r = testutil::TestRunner::deserialize(&mut de)?; + *runner_config = testutil::RunnerConfig::TestRunner(r); + if let testutil::RunnerConfig::TestRunner(x) = runner_config { + x + } else { + unreachable!() + } + } + }; let mut has_input = false; let mut has_output = false; let mut has_exec = false; @@ -63,19 +82,25 @@ fn main() -> Result<(), Error> { if parsed.debug { eprint!("runner: {:?}", runner) } - ext_set.insert(runner.suffix.clone(), runner.clone()); + ext_set.insert(runner.suffix.clone(), Arc::new(runner.clone())); } for input in &test_config.inputs { - file_format_list.insert( - (input.file_base.clone(), input.format_name.clone()), - input.clone(), - ); + let key = (input.file_base.clone(), input.format_name.clone()); + match file_format_list.get_mut(&key) { + Some(x) => {x.push(Arc::new(input.clone()));}, + None => { + file_format_list.insert( + key, + vec!{Arc::new(input.clone())}, + ); + }, + } } let mut sched = Vec::new(); let test_info = fs::read_to_string(Path::new(&parsed.test_info_file))?; let mut d2 = serde_json::Deserializer::from_str(&test_info); - let test_info = testutil::TestInfo::deserialize(&mut d2)?; - for file in &test_info.generated_files { + let mut test_info = testutil::TestInfo::deserialize(&mut d2)?; + for file in test_info.generated_files { if parsed.debug { eprintln!("file: {:?}", file); } @@ -86,7 +111,7 @@ fn main() -> Result<(), Error> { if split.len() != 2 { continue; } - let base = split[0]; + let base = split[0].to_string(); let ext = split[1]; let runner = match ext_set.get(ext) { Some(x) => x, @@ -97,12 +122,13 @@ fn main() -> Result<(), Error> { continue; } }; + let path = file.into_path(); let content = match fs::read_to_string(&path) { Ok(x) => x, Err(x) => { eprintln!("file load error: {}: {}", path, x); - continue + continue; } }; let mut d = serde_json::Deserializer::from_str(&content); @@ -113,8 +139,9 @@ fn main() -> Result<(), Error> { continue; } }; + let file = Arc::new(file); for s in &data.structs { - let key = (base.to_string(), s.clone()); + let key = (base.clone(), s.clone()); if parsed.debug { eprintln!("key: {:?}", key); } @@ -122,37 +149,55 @@ fn main() -> Result<(), Error> { if parsed.debug { eprintln!("input matched: {:?}", input); } - sched.push(TestSchedule { - runner: runner, - input: input, - file: file, - }) + for input in input { + sched.push(TestSchedule { + runner: runner.clone(), + input: input.clone(), + file: file.clone(), + }); + } } } } let mut scheduler = testutil::TestScheduler::new(); let mut failed = 0; - for s in &sched { - match scheduler.run_test_schedule(&s) { - Ok(()) => println!( - "test passed: {:?}", - s.file.into_path() + "/" + &s.input.format_name - ), + let (send, mut recv) = tokio::sync::mpsc::channel(1); + let total = sched.len(); + for s in sched { + match scheduler.run_test_schedule(&s, send.clone()) { + Ok(_) => { + println!("test {} scheduled...",s.test_name()); + } Err(x) => { - println!( - "test failed: {:?} {:?}", - s.file.into_path() + "/" + &s.input.format_name, - x - ); + println!("{}: {} {:?}","FAIL".red(), s.test_name(), x); + failed += 1; + } + } + } + drop(send); + while let Some(x) = recv.recv().await { + match x { + Ok(x) => { + println!("{}: {}","PASS".green(), x.test_name()) + } + Err((sched, err)) => { + eprintln!("{}: {}: {:?}","FAIL".red(), sched.test_name(), err); failed += 1; } } } println!( - "test finished: total/pass/failed = {}/{}/{}", - sched.len(), - sched.len() - failed, + "Result: total/{}/{} = {}/{}/{}", + "PASS".green(), + "FAIL".red(), + total, + total - failed, failed ); + if parsed.save_tmp_dir { + scheduler.print_tmp_dir(); + } else { + scheduler.remove_tmp_dir(); + } Ok(()) } diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index bc24b0c4..c5e176c6 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -1,4 +1,10 @@ -use std::{collections::HashMap, env, fs, path::PathBuf, process}; +use std::{ + collections::HashMap, + env, fs, + path::{Path, PathBuf}, + process, + sync::Arc, +}; use rand::{ self, @@ -7,6 +13,7 @@ use rand::{ use ast2rust::ast; use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; #[derive(Serialize, Deserialize, Debug)] @@ -89,27 +96,92 @@ pub struct TestInput { pub hex: bool, } +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct TestRunnerFile { + pub file: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum RunnerConfig { + TestRunner(TestRunner), + File(TestRunnerFile), +} + #[derive(Serialize, Deserialize, Clone)] pub struct TestConfig { - pub runners: Vec, + pub runners: Vec, // test input binary file pub inputs: Vec, } -pub struct TestSchedule<'a> { - pub input: &'a TestInput, - pub runner: &'a TestRunner, - pub file: &'a GeneratedFileInfo, +#[derive(Clone)] +pub struct TestSchedule { + pub input: Arc, + pub runner: Arc, + pub file: Arc, +} + +impl TestSchedule { + pub fn test_name(&self) -> String { + let res = self.file.into_path() + "/" + &self.input.format_name; + let res = res + " input:" + &self.input.binary; + let res = if self.input.hex { + res + "(hex file)" + } else { + res + }; + let res = if self.input.failure_case { + res + "(require:failure)" + } else { + res + "(require:success)" + }; + res + } +} + +#[derive(Debug)] +pub enum Error { + IO(std::io::Error), + Join(tokio::task::JoinError), + JSON(serde_json::Error), + Exec(String), + TestFail(String), +} + +impl From for Error { + fn from(s: std::io::Error) -> Error { + Error::IO(s) + } +} + +impl From for Error { + fn from(s: tokio::task::JoinError) -> Error { + Error::Join(s) + } } -pub type Error = Box; +impl From for Error { + fn from(s: serde_json::Error) -> Error { + Error::JSON(s) + } +} pub struct TestScheduler { template_files: HashMap, tmpdir: Option, - input_binaries: HashMap>, + input_binaries: HashMap<(PathBuf, bool), (PathBuf, Vec)>, +} + +fn path_str(path: &PathBuf) -> String { + match path.to_str() { + Some(x) => x.to_string(), + None => path.to_string_lossy().to_string(), + } } +type SendChan = mpsc::Sender>; + impl TestScheduler { pub fn new() -> Self { Self { @@ -125,9 +197,12 @@ impl TestScheduler { } else { let t = match fs::read_to_string(path) { Ok(x) => x, - Err(x)=> { - return Err(std::io::Error::new(std::io::ErrorKind::Other, - format!("read file error: {}: {}",path,x)).into()); + Err(x) => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("read file error: {}: {}", path, x), + ) + .into()); } }; self.template_files.insert(path.to_string(), t); @@ -144,12 +219,12 @@ impl TestScheduler { let mut col = 1; while i < input.len() { let c = input[i]; - if c == b' ' || c == b'\t' || c == b'\n'||c==b'\r' { - if c==b'\n' { - line+=1; - col=0; + if c == b' ' || c == b'\t' || c == b'\n' || c == b'\r' { + if c == b'\n' { + line += 1; + col = 0; } - col+=1; + col += 1; i += 1; continue; } @@ -168,7 +243,7 @@ impl TestScheduler { } else { return Err(std::io::Error::new( std::io::ErrorKind::Other, - format!("invalid hex string at {}:{}:{}", line,col, c as char), + format!("invalid hex string at {}:{}:{}", line, col, c as char), ) .into()); }; @@ -179,7 +254,7 @@ impl TestScheduler { pair = Some(lsb) } i += 1; - col+=1; + col += 1; } if let Some(_) = pair { Err(std::io::Error::new( @@ -192,37 +267,56 @@ impl TestScheduler { } } - fn read_input_binary(&mut self, path: &PathBuf, is_hex: bool) -> Result, Error> { - if let Some(x) = self.input_binaries.get(path) { + fn read_input_binary( + &mut self, + path: &PathBuf, + is_hex: bool, + ) -> Result<(PathBuf, Vec), Error> { + let key = (path.clone(), is_hex); + if let Some(x) = self.input_binaries.get(&key) { return Ok(x.clone()); } else { let t = match fs::read(path) { Ok(x) => x, - Err(x)=> { - return Err(std::io::Error::new(std::io::ErrorKind::Other, - format!("read file error: {:?}: {}",path,x)).into()); + Err(x) => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("read file error: {:?}: {}", path, x), + ) + .into()); } }; - let t = if is_hex { Self::compile_hex(t)? } else { t }; - self.input_binaries.insert(path.clone(), t); - Ok(self.input_binaries.get(path).unwrap().clone()) + let t = if is_hex { + let c = Self::compile_hex(t)?; + let tmp_path = self.get_tmp_dir(); + let tmp_path = tmp_path.join(path.file_name().unwrap()); + fs::write(tmp_path.clone(), &c)?; + (tmp_path, c) + } else { + (path.clone(), t) + }; + self.input_binaries.insert(key.clone(), t); + Ok(self.input_binaries.get(&key).unwrap().clone()) } } + fn gen_random() -> String { + let mut rng = rand::thread_rng(); + Alphanumeric.sample_string(&mut rng, 32) + } + fn get_tmp_dir<'a>(&'a mut self) -> PathBuf { if let Some(x) = self.tmpdir.as_ref() { x.clone() } else { let dir = env::temp_dir(); - let mut rng = rand::thread_rng(); - let random_str = Alphanumeric.sample_string(&mut rng, 32); - let dir = dir.join(random_str); + let dir = dir.join(Self::gen_random()); self.tmpdir = Some(dir); self.tmpdir.as_ref().unwrap().clone() } } - fn prepare_content<'a>(&mut self, sched: &TestSchedule<'a>) -> Result { + fn prepare_content(&mut self, sched: &TestSchedule) -> Result { // get template and replace with target let template = self.read_text_file(&sched.runner.test_template)?; let replace_with = &sched.runner.replace_struct_name; @@ -233,18 +327,19 @@ impl TestScheduler { Ok(instance) } - fn create_test_dir<'a>(&mut self, sched: &TestSchedule<'a>) -> Result { + fn create_test_dir(&mut self, sched: &TestSchedule) -> Result { let tmp_dir = self.get_tmp_dir(); let tmp_dir = tmp_dir.join(&sched.file.base); let tmp_dir = tmp_dir.join(&sched.input.format_name); let tmp_dir = tmp_dir.join(&sched.file.suffix); + let tmp_dir = tmp_dir.join(Self::gen_random()); fs::create_dir_all(&tmp_dir)?; Ok(tmp_dir) } - fn create_input_file<'a>( + fn create_input_file( &mut self, - sched: &TestSchedule<'a>, + sched: &TestSchedule, tmp_dir: &PathBuf, instance: String, ) -> Result<(PathBuf, PathBuf), Error> { @@ -254,9 +349,9 @@ impl TestScheduler { Ok((input_file, output_file)) } - fn replace_cmd<'a>( + fn replace_cmd( cmd: &mut Vec, - sched: &TestSchedule<'a>, + sched: &TestSchedule, tmp_dir: &PathBuf, input: &PathBuf, output: &PathBuf, @@ -283,9 +378,8 @@ impl TestScheduler { } } - fn exec_cmd<'a>( - &mut self, - sched: &TestSchedule<'a>, + async fn exec_cmd( + sched: &TestSchedule, base: &Vec, tmp_dir: &PathBuf, input: &PathBuf, @@ -295,10 +389,15 @@ impl TestScheduler { ) -> Result { let mut cmd = base.clone(); Self::replace_cmd(&mut cmd, sched, tmp_dir, input, output, exec); - let mut r = process::Command::new(&cmd[0]); + let mut r = tokio::process::Command::new(&cmd[0]); r.args(&cmd[1..]); - let done = r.output()?; - let code = done.status.code(); + let done = match r.status().await { + Ok(x) => x, + Err(x) => { + return Err(Error::Exec(format!("exec error: {:?}: {}", cmd, x))); + } + }; + let code = done.code(); match code { Some(0) => return Ok(true), status => { @@ -307,98 +406,155 @@ impl TestScheduler { return Ok(false); } } - let stderr_str = String::from_utf8_lossy(&done.stderr); - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("process exit with {:?}\n{}", status, stderr_str), - ) - .into()); + return Err(Error::Exec(format!("process exit with {:?}", status))); } }; } - pub fn run_test_schedule<'a>(&mut self, sched: &TestSchedule<'a>) -> Result<(), Error> { - let tmp_dir = self.create_test_dir(sched)?; - let instance = self.prepare_content(sched)?; + pub fn run_test_schedule_impl( + sched: TestSchedule, + send: SendChan, + tmp_dir: PathBuf, + input: PathBuf, + input_path: PathBuf, + output: PathBuf, + input_binary: Vec, + ) -> Result, Error> { + let proc = async move { + // build test + match Self::exec_cmd( + &sched, + &sched.runner.build_command, + &tmp_dir, + &input, + &output, + None, + true, + ) + .await + { + Ok(_) => {} + Err(x) => return Err((sched, x)), + }; - let (input, output) = self.create_input_file(sched, &tmp_dir, instance)?; + let exec = output; - // build test - self.exec_cmd( - &sched, - &sched.runner.build_command, - &tmp_dir, - &input, - &output, - None, - true, - )?; + let output = tmp_dir.join("output.bin"); - let exec = output; + // run test + let status = match Self::exec_cmd( + &sched, + &sched.runner.run_command, + &tmp_dir, + &input_path, + &output, + Some(&exec), + false, + ) + .await + { + Ok(x) => x, + Err(x) => return Err((sched, x)), + }; - let output = tmp_dir.join("output.bin"); + let expect = !sched.input.failure_case; - let input_binary_name: PathBuf = sched.input.binary.clone().into(); + if status != expect { + return Err(( + sched, + Error::TestFail(format!("test failed: expect {} but got {}", expect, status)), + )); + } - let input_binary = self.read_input_binary(&input_binary_name, sched.input.hex)?; // check input is valid + if sched.input.failure_case { + return Ok(sched); // skip output check + } - // run test - let status = self.exec_cmd( - &sched, - &sched.runner.run_command, - &tmp_dir, - &input_binary_name, - &output, - Some(&exec), - false, - )?; + let output = match tokio::fs::read(&output).await { + Ok(x) => x, + Err(x) => { + return Err(( + sched, + Error::TestFail(format!("test output cannot load: {}", x)), + )) + } + }; - let expect = !sched.input.failure_case; + let min_size = if input_binary.len() < output.len() { + input_binary.len() + } else { + output.len() + }; - if status != expect { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("test failed: expect {} but got {}", expect, status), - ) - .into()); - } + let mut diff = Vec::new(); - let output = fs::read(&output)?; // check output is valid + for i in 0..min_size { + if input_binary[i] != output[i] { + diff.push((i, Some(input_binary[i]), Some(output[i]))); + } + } - let min_size = if input_binary.len() < output.len() { - input_binary.len() - } else { - output.len() + if input_binary.len() != output.len() { + if input_binary.len() > output.len() { + diff.push((output.len(), None, Some(output[output.len()]))); + } else { + diff.push(( + input_binary.len(), + Some(input_binary[input_binary.len()]), + None, + )); + } + } + + if !diff.is_empty() { + let mut debug = format!("input and output is different\n"); + for (i, a, b) in diff { + debug += &format!("{}: {:02x?} != {:02x?}\n", i, a, b); + } + return Err((sched, Error::TestFail(debug))); + } + + Ok(sched) }; + let proc = async move { + let r = proc.await; + send.send(r).await.unwrap(); + }; + Ok(tokio::spawn(proc)) + } - let mut diff = Vec::new(); + pub fn run_test_schedule( + &mut self, + sched: &TestSchedule, + send: SendChan, + ) -> Result, Error> { + let tmp_dir = self.create_test_dir(sched)?; + let instance = self.prepare_content(sched)?; - for i in 0..min_size { - if input_binary[i] != output[i] { - diff.push((i, Some(input_binary[i]), Some(output[i]))); - } - } + let (input, output) = self.create_input_file(sched, &tmp_dir, instance)?; + let input_path: PathBuf = sched.input.binary.clone().into(); + let (input_path, input_binary) = self.read_input_binary(&input_path, sched.input.hex)?; + + Self::run_test_schedule_impl( + sched.clone(), + send, + tmp_dir, + input, + input_path, + output, + input_binary, + ) + } - if input_binary.len() != output.len() { - if input_binary.len() > output.len() { - diff.push((output.len(), None, Some(output[output.len()]))); - } else { - diff.push(( - input_binary.len(), - Some(input_binary[input_binary.len()]), - None, - )); - } + pub fn remove_tmp_dir(self) { + if let Some(dir) = self.tmpdir { + fs::remove_dir_all(dir).unwrap(); } + } - if !diff.is_empty() { - let mut debug = format!("input and output is different\n"); - for (i, a, b) in diff { - debug += &format!("{}: {:02x?} != {:02x?}\n", i, a, b); - } - return Err(std::io::Error::new(std::io::ErrorKind::Other, debug).into()); + pub fn print_tmp_dir(self) { + if let Some(dir) = self.tmpdir { + println!("tmp directory is {}", path_str(&dir)); } - - Ok(()) } } diff --git a/src/tool/json2ts/generate.cpp b/src/tool/json2ts/generate.cpp index a592436f..90f01893 100644 --- a/src/tool/json2ts/generate.cpp +++ b/src/tool/json2ts/generate.cpp @@ -95,6 +95,190 @@ namespace json2ts { return "any"; } + void write_union_field(ast::StructUnionType* u, const std::shared_ptr& field, brgen::writer::Writer& wt) { + bool first = true; + auto anonymous_field = field->ident->ident; + str.map_ident(field->ident, prefix, ".", anonymous_field); + wt.writeln(anonymous_field, ": "); + auto dot = "."; + auto p = std::move(prefix); + if (typescript) { + prefix = brgen::concat("(", p, dot, anonymous_field, " as any)"); + } + else { + prefix = brgen::concat(p, dot, anonymous_field); + } + size_t i = 0; + for (auto s : u->structs) { + if (!first) { + wt.writeln("|"); + } + + brgen::writer::Writer tmpw; + + write_struct_type(tmpw, s); + + first = false; + wt.write_unformatted(tmpw.out()); + char prev_char = 0; + std::erase_if(tmpw.out(), [&](char c) { + if (prev_char == ' ' && c == ' ') { + return true; + } + prev_char = c; + return c == '\n'; + }); + anonymous_types[s] = {anonymous_field, tmpw.out(), field}; + + if (typescript) { + auto& name = field->belong.lock()->ident->ident; + w.writeln("export interface ", name, "_", anonymous_field, "_", brgen::nums(i), " ", tmpw.out(), ";"); + i++; + } + + wt.writeln(); + } + std::swap(prefix, p); + if (!u->exhaustive) { + if (!first) { + wt.writeln("|"); + } + wt.write("undefined"); + } + wt.writeln(";"); + for (auto& f : u->union_fields) { + auto union_field = f.lock(); + auto union_ty = ast::as(union_field->field_type); + str.map_ident(union_field->ident, p, ".", union_field->ident->ident); + auto fmt = ast::as(union_field->belong.lock()); + if (union_ty->common_type) { + // getter + w.write("function ", fmt->ident->ident, "_get_", union_field->ident->ident, "(obj"); + if (typescript) { + w.write(" :", fmt->ident->ident); + } + w.writeln(") {"); + { + auto indent = w.indent_scope(); + std::string cond; + if (auto c = union_ty->cond.lock()) { + cond = str.to_string(c); + } + else { + cond = "true"; + } + bool first = false; + bool els = false; + for (auto& cand : union_ty->candidates) { + auto cond1 = cand->cond.lock(); + if (ast::is_any_range(cond1)) { + els = true; + if (!first) { + w.write("else "); + } + w.writeln("{"); + if (auto f = cand->field.lock()) { + w.writeln("return ", str.to_string(f->ident), ";"); + } + else { + w.writeln("return null;"); + } + w.writeln("}"); + } + else { + if (!first) { + w.write("else "); + } + auto conds = str.to_string(cond1); + w.writeln("if (", cond, "==", conds, ") {"); + if (auto f = cand->field.lock()) { + w.writeln("return ", str.to_string(f->ident), ";"); + } + else { + w.writeln("return null;"); + } + w.writeln("}"); + } + } + if (!els) { + w.writeln("return null;"); + } + } + w.writeln("}"); + + // setter + w.write("function ", fmt->ident->ident, "_set_", union_field->ident->ident, "("); + if (typescript) { + auto ty = get_type(union_ty->common_type); + w.write("obj :Partial<", fmt->ident->ident, ">,value :", ty); + } + else { + w.write("obj,value"); + } + w.writeln(") {"); + { + auto indent = w.indent_scope(); + std::string cond; + if (auto c = union_ty->cond.lock()) { + cond = str.to_string(c); + } + else { + cond = "true"; + } + bool first = false; + bool els = false; + auto write_set = [&](auto& cand) { + if (auto f = cand->field.lock()) { + auto& typ = anonymous_types[f->belong_struct.lock()].type; + auto base_field = str.to_string(field->ident); + w.writeln("if(!", base_field, ") {"); + { + auto indent = w.indent_scope(); + w.write(base_field, " = {}"); + if (typescript) { + w.write(" as ", typ); + } + w.writeln(";"); + } + w.writeln("}"); + w.writeln(str.to_string(f->ident), "= value;"); + w.writeln("return true;"); + } + else { + w.writeln("return false;"); + } + }; + for (auto& cand : union_ty->candidates) { + auto cond1 = cand->cond.lock(); + if (ast::is_any_range(cond1)) { + els = true; + if (!first) { + w.write("else "); + } + w.writeln("{"); + write_set(cand); + w.writeln("}"); + } + else { + if (!first) { + w.write("else "); + } + auto conds = str.to_string(cond1); + w.writeln("if (", cond, "==", conds, ") {"); + write_set(cand); + w.writeln("}"); + } + if (!els) { + w.writeln("return false;"); + } + } + } + w.writeln("}"); + } + } + return; + } + void write_field(brgen::writer::Writer& wt, std::vector>& bit_fields, const std::shared_ptr& field) { if (!field->ident) { ast::tool::set_tmp_field_ident(get_seq(), field, "anonymous_"); @@ -131,60 +315,7 @@ namespace json2ts { return; } if (auto u = ast::as(typ)) { - bool first = true; - auto anonymous_field = field->ident->ident; - str.map_ident(field->ident, prefix, ".", anonymous_field); - wt.writeln(anonymous_field, ": "); - auto dot = "."; - auto p = std::move(prefix); - if (typescript) { - prefix = brgen::concat("(", p, dot, anonymous_field, " as any)"); - } - else { - prefix = brgen::concat(p, dot, anonymous_field); - } - size_t i = 0; - for (auto s : u->structs) { - if (!first) { - wt.writeln("|"); - } - - brgen::writer::Writer tmpw; - - write_struct_type(tmpw, s); - - first = false; - wt.write_unformatted(tmpw.out()); - char prev_char = 0; - std::erase_if(tmpw.out(), [&](char c) { - if (prev_char == ' ' && c == ' ') { - return true; - } - prev_char = c; - return c == '\n'; - }); - anonymous_types[s] = {anonymous_field, tmpw.out(), field}; - - if (typescript) { - auto& name = field->belong.lock()->ident->ident; - w.writeln("export interface ", name, "_", anonymous_field, "_", brgen::nums(i), " ", tmpw.out(), ";"); - i++; - } - - wt.writeln(); - } - std::swap(prefix, p); - if (!u->exhaustive) { - if (!first) { - wt.writeln("|"); - } - wt.write("undefined"); - } - wt.writeln(";"); - for (auto& f : u->union_fields) { - auto field = f.lock(); - str.map_ident(field->ident, p, ".", field->ident->ident); - } + write_union_field(u, field, wt); return; } auto type = get_type(typ); diff --git a/testkit/cmptest.json b/testkit/cmptest.json index 066181c1..3e746814 100644 --- a/testkit/cmptest.json +++ b/testkit/cmptest.json @@ -6,23 +6,18 @@ "file_base": "websocket", "failure_case": false, "hex": true + }, + { + "binary": "./example/wire_data/websocket.dat", + "format_name": "Frame", + "file_base": "websocket", + "failure_case": true, + "hex": false } ], "runners": [ { - "suffix": "hpp", - "test_template": "./testkit/cpp/test_template.cpp", - "replace_file_name": "stub.hpp", - "replace_struct_name": "TEST_CLASS", - "build_input_name": "test.cpp", - "build_output_name": "test.exe", - "build_command": [ - "python", - "./testkit/cpp/setup.py", - "$INPUT", - "$OUTPUT" - ], - "run_command": ["$EXEC", "$INPUT", "$OUTPUT"] + "file": "./testkit/cpp/config.json" } ] } diff --git a/testkit/cpp/config.json b/testkit/cpp/config.json new file mode 100644 index 00000000..9544d6d5 --- /dev/null +++ b/testkit/cpp/config.json @@ -0,0 +1,17 @@ +{ + "suffix": "hpp", + "test_template": "./testkit/cpp/test_template.cpp", + "replace_file_name": "stub.hpp", + "replace_struct_name": "TEST_CLASS", + "build_input_name": "test.cpp", + "build_output_name": "test.exe", + "build_command": [ + "python", + "./testkit/cpp/setup.py", + "$INPUT", + "$OUTPUT", + "$ORIGIN", + "$TMPDIR" + ], + "run_command": ["$EXEC", "$INPUT", "$OUTPUT"] +} \ No newline at end of file diff --git a/testkit/cpp/setup.py b/testkit/cpp/setup.py index edb93280..bbf7ef1b 100644 --- a/testkit/cpp/setup.py +++ b/testkit/cpp/setup.py @@ -1,35 +1,61 @@ import sys +import os +import subprocess as sp +import pathlib as pl +import shutil +import platform as plt INPUT = sys.argv[1] OUTPUT = sys.argv[2] +ORIGIN = sys.argv[3] +TMPDIR = sys.argv[4] -import os +with open(INPUT, "r") as f: + ABS_ORIGIN = pl.Path(ORIGIN).absolute().as_posix() + REPLACED_INPUT = f.read().replace(ORIGIN, ABS_ORIGIN) + +with open(INPUT, "w") as f: + f.write(REPLACED_INPUT) FUTILS_DIR = os.environ.get("FUTILS_DIR") if FUTILS_DIR is None: print("FUTILS_DIR not set", file=sys.stderr) exit(2) -import subprocess as sp # run compiler for the test # add link directory $FUTILS_DIR/lib and -lfutils # compiler output will be redirected to stdout and stderr CC = "clang++" - -sp.call( - [ - CC, - "-std=c++20", - "-I", - FUTILS_DIR + "/include", - "-L", - FUTILS_DIR + "/lib", - "-lfutils", - INPUT, - "-o", - OUTPUT, - ], +CMDLINE = [ + CC, + "-std=c++20", + "-I", + FUTILS_DIR + "/src/include", + "-L", + FUTILS_DIR + "/lib", + "-lfutils", + INPUT, + "-o", + OUTPUT, +] +print(f"Compiling {INPUT} to {OUTPUT} with {CMDLINE} ") +code = sp.call( + CMDLINE, stdout=sys.stdout, stderr=sys.stderr, ) +if code != 0: + exit(code) + +suffix = ".so" + +match plt.system(): + case "Linux": + suffix = ".so" + case "Darwin": + suffix = ".dylib" + case "Windows": + suffix = ".dll" + +shutil.copyfile(FUTILS_DIR + "/lib/libfutils" + suffix, TMPDIR + "/libfutils" + suffix) diff --git a/testkit/cpp/test_template.cpp b/testkit/cpp/test_template.cpp index c5b604d0..27401772 100644 --- a/testkit/cpp/test_template.cpp +++ b/testkit/cpp/test_template.cpp @@ -59,7 +59,7 @@ int main(int argc, char** argv) { return e; } if (!r.empty()) { - cout << "Failed to decode file " << argv[1] << ";" << r.remain().size() << " bytes left\n"; + cout << "Failed to decode file " << argv[1] << "; " << r.remain().size() << " bytes left\n"; return 1; } std::string s; From 4c5c6e3621d7c21652891297ead319542b874859 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Sun, 28 Apr 2024 16:28:53 +0900 Subject: [PATCH 34/50] remove needless suffix --- .vscode/launch.json | 15 +++------------ src/tool/cmptest/src/testutil.rs | 1 - testkit/cpp/setup.py | 1 + 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0c9da65d..fc20f064 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -57,19 +57,10 @@ "name": "Debug json2cpp", "type": "cppvsdbg", "request": "launch", - "program": "${workspaceRoot}/tool/json2cpp2", + "program": "C:/Users/ukfco/AppData/Local/Temp/gfovKCSlcGBvX5zCejAJQziajuBYOwqd/websocket.hpp/Frame/.json/e8NZ2jxzQxr6Woq75m1fNJDoBg70rUEh/test.exe", "args": [ - "<", - "ignore/request.bin", - ">", - "ignore/response.bin", - //"ignore/sample.json", - //"-f", - /*"-r", - "--call", "Test2.decode", - "--binary","example/vm_test/varint.dat", - "--hex", - */ + "./example/wire_data/websocket.dat", + "./ignore/output.bin" ], "stopAtEntry": false, "cwd": "${workspaceRoot}", diff --git a/src/tool/cmptest/src/testutil.rs b/src/tool/cmptest/src/testutil.rs index c5e176c6..0b315f3d 100644 --- a/src/tool/cmptest/src/testutil.rs +++ b/src/tool/cmptest/src/testutil.rs @@ -331,7 +331,6 @@ impl TestScheduler { let tmp_dir = self.get_tmp_dir(); let tmp_dir = tmp_dir.join(&sched.file.base); let tmp_dir = tmp_dir.join(&sched.input.format_name); - let tmp_dir = tmp_dir.join(&sched.file.suffix); let tmp_dir = tmp_dir.join(Self::gen_random()); fs::create_dir_all(&tmp_dir)?; Ok(tmp_dir) diff --git a/testkit/cpp/setup.py b/testkit/cpp/setup.py index bbf7ef1b..346abc25 100644 --- a/testkit/cpp/setup.py +++ b/testkit/cpp/setup.py @@ -38,6 +38,7 @@ INPUT, "-o", OUTPUT, + "-g", ] print(f"Compiling {INPUT} to {OUTPUT} with {CMDLINE} ") code = sp.call( From e5e2726924e8c7bbc6b8b5b0d1fe4fed8fb83d4e Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 00:01:11 +0900 Subject: [PATCH 35/50] can build on windows test --- .vscode/launch.json | 6 +++--- build.sh | 7 +++++-- src/tool/cmptest/src/main.rs | 9 ++++++--- testkit/cpp/setup.py | 6 ++++++ testkit/cpp/test_template.cpp | 9 ++++++++- 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index fc20f064..e9299c1a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -57,10 +57,10 @@ "name": "Debug json2cpp", "type": "cppvsdbg", "request": "launch", - "program": "C:/Users/ukfco/AppData/Local/Temp/gfovKCSlcGBvX5zCejAJQziajuBYOwqd/websocket.hpp/Frame/.json/e8NZ2jxzQxr6Woq75m1fNJDoBg70rUEh/test.exe", + "program": "./tool/json2cpp", "args": [ - "./example/wire_data/websocket.dat", - "./ignore/output.bin" + "-f", + "./example/websocket.bgn", ], "stopAtEntry": false, "cwd": "${workspaceRoot}", diff --git a/build.sh b/build.sh index bd1aefa5..2626e908 100644 --- a/build.sh +++ b/build.sh @@ -1,7 +1,10 @@ #!/bin/bash # TODO(on-keyday): on macos, S2J_LIB=1 is not work well, so not set S2J_LIB - -export S2J_LIB=1 +if [ "$(uname)" == "Darwin" ]; then + export S2J_LIB=0 +else + export S2J_LIB=1 +fi export BRGEN_RUST_ENABLED=1 diff --git a/src/tool/cmptest/src/main.rs b/src/tool/cmptest/src/main.rs index 5ba4837e..5264781b 100644 --- a/src/tool/cmptest/src/main.rs +++ b/src/tool/cmptest/src/main.rs @@ -187,11 +187,11 @@ async fn main() -> Result<(), Error> { } } println!( - "Result: total/{}/{} = {}/{}/{}", - "PASS".green(), - "FAIL".red(), + "Result: Total:{} {}:{} {}:{}", total, + "PASS".green(), total - failed, + "FAIL".red(), failed ); if parsed.save_tmp_dir { @@ -199,5 +199,8 @@ async fn main() -> Result<(), Error> { } else { scheduler.remove_tmp_dir(); } + if failed > 0 { + return Err(testutil::Error::TestFail("some tests failed".to_string()).into()); + } Ok(()) } diff --git a/testkit/cpp/setup.py b/testkit/cpp/setup.py index 346abc25..d772fff4 100644 --- a/testkit/cpp/setup.py +++ b/testkit/cpp/setup.py @@ -40,6 +40,12 @@ OUTPUT, "-g", ] + +# use dynamic linking crt on windows +if plt.system() == "Windows": + CMDLINE.append("-nostdlib") + CMDLINE.append("-fms-runtime-lib=dll_dbg") + print(f"Compiling {INPUT} to {OUTPUT} with {CMDLINE} ") code = sp.call( CMDLINE, diff --git a/testkit/cpp/test_template.cpp b/testkit/cpp/test_template.cpp index 27401772..503461f3 100644 --- a/testkit/cpp/test_template.cpp +++ b/testkit/cpp/test_template.cpp @@ -5,7 +5,14 @@ #include #include #include -auto& cout = futils::wrap::cerr_wrap(); +#define _DEBUG 1 +#include +#include +#include +auto& cout = []() -> decltype(auto) { + futils::test::set_alloc_hook(true); + return futils::wrap::cerr_wrap(); +}(); int do_encode(char** argv, auto& t, auto& w) { if constexpr (std::is_same_v) { From 2f78928613572eeda4716105168332777d63a4ab Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 00:06:11 +0900 Subject: [PATCH 36/50] add cmptest --- .github/workflows/brgen-test.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/brgen-test.yaml b/.github/workflows/brgen-test.yaml index b4626b8a..7cf23bb5 100644 --- a/.github/workflows/brgen-test.yaml +++ b/.github/workflows/brgen-test.yaml @@ -26,7 +26,7 @@ jobs: path: /tmp/brgen - name: Install Dependencies run: sudo apt-get install jq libc++-dev ninja-build -y - - name: Run Brgen and Test + - name: Run Brgen run: | chmod -R +x /tmp/brgen cd /tmp/brgen @@ -35,9 +35,12 @@ jobs: ldd /tmp/brgen/tool/libfutils.so /tmp/brgen/tool/src2json --version /tmp/brgen/tool/brgen -debug + - name: Clone Utils + run: | git clone https://github.com/on-keyday/utils.git - cd utils - . build shared Debug futils - cd .. - export FUTILS_DIR=/tmp/brgen/utils - # ./make_cpp_tests.sh ./ignore/example/test_info.json run + mkdir utils/lib + cp /tmp/brgen/tool/libfutils.so utils/lib + - name: Test + run: | + cd /tmp/brgen + tool/cmptest -f ignore/example/test_info.json -c testkit/cmptest.json From b12866dd1a43d156d1d0c983e09d03c23c3efb87 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 00:21:38 +0900 Subject: [PATCH 37/50] fix forgotten env --- .github/workflows/brgen-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/brgen-test.yaml b/.github/workflows/brgen-test.yaml index 7cf23bb5..8982b318 100644 --- a/.github/workflows/brgen-test.yaml +++ b/.github/workflows/brgen-test.yaml @@ -43,4 +43,4 @@ jobs: - name: Test run: | cd /tmp/brgen - tool/cmptest -f ignore/example/test_info.json -c testkit/cmptest.json + FUTILS_DIR=./utils tool/cmptest -f ignore/example/test_info.json -c testkit/cmptest.json From dafdbde537fd1fa4403699ae96b3666b2a62669a Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 00:48:14 +0900 Subject: [PATCH 38/50] fix --- .github/workflows/brgen-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/brgen-test.yaml b/.github/workflows/brgen-test.yaml index 8982b318..c6334075 100644 --- a/.github/workflows/brgen-test.yaml +++ b/.github/workflows/brgen-test.yaml @@ -43,4 +43,4 @@ jobs: - name: Test run: | cd /tmp/brgen - FUTILS_DIR=./utils tool/cmptest -f ignore/example/test_info.json -c testkit/cmptest.json + FUTILS_DIR=/tmp/brgen/utils tool/cmptest -f ignore/example/test_info.json -c testkit/cmptest.json From 9acfd1ca8fe29ef52ec52f98bb569616424b1cd7 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 01:30:17 +0900 Subject: [PATCH 39/50] fix --- src/tool/json2ts/generate.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tool/json2ts/generate.cpp b/src/tool/json2ts/generate.cpp index 90f01893..b624abf3 100644 --- a/src/tool/json2ts/generate.cpp +++ b/src/tool/json2ts/generate.cpp @@ -459,7 +459,7 @@ namespace json2ts { w.writeln("if(w.resizeLimit !== undefined && new_size > w.resizeLimit) {"); { auto s = w.indent_scope(); - w.writeln("throw new Error(`new buffer size is greater than w.resizeLimit for ", field_name, " required={new_size} limit={r.resizeLimit}`);"); + w.writeln("throw new Error(`new buffer size is greater than w.resizeLimit for ", field_name, " required=${new_size} limit=${w.resizeLimit}`);"); } w.writeln("}"); w.writeln("const new_buffer = new ArrayBuffer(w.view.byteLength + (", len, "));"); @@ -742,7 +742,7 @@ namespace json2ts { w.writeln("if(", len, "<", "0", ") {"); { auto s = w.indent_scope(); - w.writeln("throw new Error(`invalid buffer size for ", err_ident->ident, " underflow={", len, "}`);"); + w.writeln("throw new Error(`invalid buffer size for ", err_ident->ident, " underflow=${", len, "}`);"); } w.writeln("}"); } From 958fd6bc612c0874a475e4091b57f3bdf1bf6d50 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 02:33:41 +0900 Subject: [PATCH 40/50] remove needless linker --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 718c7a4e..42f26900 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(NOT "$ENV{FUTILS_TARGET_TRIPLE}" STREQUAL "") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target=$ENV{FUTILS_TARGET_TRIPLE}") endif() - set(CMAKE_CXX_FLAGS "-fuse-ld=lld") endif() if(UNIX) From 207cff8e0eddef0c052c213ea84b1719d5a9220f Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:20:57 +0900 Subject: [PATCH 41/50] use libc++ for test --- .github/workflows/brgen-test.yaml | 4 +++- testkit/cpp/setup.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/brgen-test.yaml b/.github/workflows/brgen-test.yaml index c6334075..06c5590a 100644 --- a/.github/workflows/brgen-test.yaml +++ b/.github/workflows/brgen-test.yaml @@ -37,10 +37,12 @@ jobs: /tmp/brgen/tool/brgen -debug - name: Clone Utils run: | + cd /tmp/brgen git clone https://github.com/on-keyday/utils.git mkdir utils/lib - cp /tmp/brgen/tool/libfutils.so utils/lib + cp tool/libfutils.so utils/lib - name: Test run: | cd /tmp/brgen + . ./link_path.sh FUTILS_DIR=/tmp/brgen/utils tool/cmptest -f ignore/example/test_info.json -c testkit/cmptest.json diff --git a/testkit/cpp/setup.py b/testkit/cpp/setup.py index d772fff4..120c3ab7 100644 --- a/testkit/cpp/setup.py +++ b/testkit/cpp/setup.py @@ -45,6 +45,8 @@ if plt.system() == "Windows": CMDLINE.append("-nostdlib") CMDLINE.append("-fms-runtime-lib=dll_dbg") +else: + CMDLINE.append("-stdlib=libc++") print(f"Compiling {INPUT} to {OUTPUT} with {CMDLINE} ") code = sp.call( From 8fef25f06a88f822028b6c2d3e85d23f298f08e1 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 17:51:34 +0900 Subject: [PATCH 42/50] fix parser bug --- .vscode/launch.json | 4 ++-- src/core/ast/parse.cpp | 1 + test-brgen.json | 37 +++++++++++++++++------------- web/doc/content/docs/exec_model.md | 6 ++--- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index e9299c1a..a6a0a035 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -57,10 +57,10 @@ "name": "Debug json2cpp", "type": "cppvsdbg", "request": "launch", - "program": "./tool/json2cpp", + "program": "${workspaceRoot}/tool/json2ts", "args": [ "-f", - "./example/websocket.bgn", + "./ignore/sample.json", ], "stopAtEntry": false, "cwd": "${workspaceRoot}", diff --git a/src/core/ast/parse.cpp b/src/core/ast/parse.cpp index e2ba8dea..79bbe0e8 100644 --- a/src/core/ast/parse.cpp +++ b/src/core/ast/parse.cpp @@ -310,6 +310,7 @@ namespace brgen::ast { field->ident = ident; ident->base = field; field->field_type = union_type; + field->belong = state.current_member(); union_type->base_type = type; size_t cand_i = 0; for (auto& c : v) { diff --git a/test-brgen.json b/test-brgen.json index 11711ae4..06f8b846 100644 --- a/test-brgen.json +++ b/test-brgen.json @@ -16,22 +16,9 @@ "test_info_output": "./ignore/example/test_info.json", "output": [ { - "generator": "./tool/json2go", - "output_dir": "./ignore/example/go/", - "args": [ - "-map-word", - "Id=ID", - "-map-word", - "IDentifier=Identifier", - "-map-word", - "Tcpsegment=TCPSegment", - "-map-word", - "Udpdatagram=UDPDatagram", - "-map-word", - "Udpheader=UDPHeader", - "-map-word", - "icmp=ICMP" - ] + "generator": "./tool/json2ts", + "output_dir": "./ignore/example/ts/", + "args": [] } ], "garbage": [ @@ -65,6 +52,24 @@ "generator": "./tool/json2ts", "output_dir": "./ignore/example/ts/", "args": ["--javascript"] + }, + { + "generator": "./tool/json2go", + "output_dir": "./ignore/example/go/", + "args": [ + "-map-word", + "Id=ID", + "-map-word", + "IDentifier=Identifier", + "-map-word", + "Tcpsegment=TCPSegment", + "-map-word", + "Udpdatagram=UDPDatagram", + "-map-word", + "Udpheader=UDPHeader", + "-map-word", + "icmp=ICMP" + ] } ] } diff --git a/web/doc/content/docs/exec_model.md b/web/doc/content/docs/exec_model.md index ada9f93f..14a1d42b 100644 --- a/web/doc/content/docs/exec_model.md +++ b/web/doc/content/docs/exec_model.md @@ -306,8 +306,8 @@ TODO(on-keyday): 続きを書く 本言語ではビットフィールドが特別扱いされないという事になっている。 まあ筆者も他の類似ツールが`bitfield(16) { n :1, reserved :15 }`みたいな書き方しているのや capnproto とかいうフォーマットのやつがこの言語の書き方では -うまく表せないなあと気づいたときに返るべきかと思ったりはしているが、 -現状こんな +うまく表せないなあと気づいたとき変えるべきかと思ったりはしているが、 +現状こんな感じである。 ``` format E: @@ -329,4 +329,4 @@ format F: d :u4 ``` -この場合は +この場合は a が 0x03,b が 0x0C,c が 0xf0 である。 From 6bf9f98489aae345b9fbe4da7b963ce1b2a85ec4 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 20:09:12 +0900 Subject: [PATCH 43/50] fix ts generation --- src/tool/json2ts/generate.cpp | 81 ++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/src/tool/json2ts/generate.cpp b/src/tool/json2ts/generate.cpp index b624abf3..28e983e8 100644 --- a/src/tool/json2ts/generate.cpp +++ b/src/tool/json2ts/generate.cpp @@ -152,23 +152,39 @@ namespace json2ts { str.map_ident(union_field->ident, p, ".", union_field->ident->ident); auto fmt = ast::as(union_field->belong.lock()); if (union_ty->common_type) { + std::string cond; + if (auto c = union_ty->cond.lock()) { + cond = str.to_string(c); + } + else { + cond = "true"; + } // getter - w.write("function ", fmt->ident->ident, "_get_", union_field->ident->ident, "(obj"); + w.write("export function ", fmt->ident->ident, "_get_", union_field->ident->ident, "(obj"); if (typescript) { w.write(" :", fmt->ident->ident); } w.writeln(") {"); { auto indent = w.indent_scope(); - std::string cond; - if (auto c = union_ty->cond.lock()) { - cond = str.to_string(c); - } - else { - cond = "true"; - } - bool first = false; + + bool first = true; bool els = false; + auto write_get = [&](auto& cand) { + auto indent = w.indent_scope(); + if (auto f = cand->field.lock()) { + w.writeln("if(", str.to_string(f->ident), " !== undefined) {"); + { + auto indent = w.indent_scope(); + w.write("return ", str.to_string(f->ident)); + if (typescript) { + w.write(" as ", get_type(f->field_type)); + } + w.writeln(";"); + } + w.writeln("}"); + } + }; for (auto& cand : union_ty->candidates) { auto cond1 = cand->cond.lock(); if (ast::is_any_range(cond1)) { @@ -177,12 +193,7 @@ namespace json2ts { w.write("else "); } w.writeln("{"); - if (auto f = cand->field.lock()) { - w.writeln("return ", str.to_string(f->ident), ";"); - } - else { - w.writeln("return null;"); - } + write_get(cand); w.writeln("}"); } else { @@ -190,13 +201,8 @@ namespace json2ts { w.write("else "); } auto conds = str.to_string(cond1); - w.writeln("if (", cond, "==", conds, ") {"); - if (auto f = cand->field.lock()) { - w.writeln("return ", str.to_string(f->ident), ";"); - } - else { - w.writeln("return null;"); - } + w.writeln("if (", cond, "=== (", conds, ")) {"); + write_get(cand); w.writeln("}"); } } @@ -207,7 +213,7 @@ namespace json2ts { w.writeln("}"); // setter - w.write("function ", fmt->ident->ident, "_set_", union_field->ident->ident, "("); + w.write("export function ", fmt->ident->ident, "_set_", union_field->ident->ident, "("); if (typescript) { auto ty = get_type(union_ty->common_type); w.write("obj :Partial<", fmt->ident->ident, ">,value :", ty); @@ -218,16 +224,10 @@ namespace json2ts { w.writeln(") {"); { auto indent = w.indent_scope(); - std::string cond; - if (auto c = union_ty->cond.lock()) { - cond = str.to_string(c); - } - else { - cond = "true"; - } - bool first = false; + bool first = true; bool els = false; auto write_set = [&](auto& cand) { + auto indent = w.indent_scope(); if (auto f = cand->field.lock()) { auto& typ = anonymous_types[f->belong_struct.lock()].type; auto base_field = str.to_string(field->ident); @@ -253,24 +253,27 @@ namespace json2ts { if (ast::is_any_range(cond1)) { els = true; if (!first) { - w.write("else "); + w.writeln("else {"); + write_set(cand); + w.writeln("}"); + } + else { + write_set(cand); } - w.writeln("{"); - write_set(cand); - w.writeln("}"); } else { if (!first) { w.write("else "); } auto conds = str.to_string(cond1); - w.writeln("if (", cond, "==", conds, ") {"); + w.writeln("if (", cond, "=== (", conds, ")) {"); write_set(cand); w.writeln("}"); } - if (!els) { - w.writeln("return false;"); - } + first = false; + } + if (!els) { + w.writeln("return false;"); } } w.writeln("}"); From 9b744c7a705cc05c1d7dd3de877dc129d088ad5f Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 20:23:38 +0900 Subject: [PATCH 44/50] add formats --- example/webauthn.bgn | 90 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 3 deletions(-) diff --git a/example/webauthn.bgn b/example/webauthn.bgn index 3eea634b..4e2feeed 100644 --- a/example/webauthn.bgn +++ b/example/webauthn.bgn @@ -37,7 +37,91 @@ enum Algorithm: ES256 = -7 RS256 = -257 +enum AttestationConveyancePreference: + :u8 + none = 0 + indirect = 1 + direct = 2 + enterprise = 3 + +format AttestationFormat: + fmt_len :u8 + fmt :[fmt_len]u8 + +enum AuthenticatorAttachment: + :u8 + platform = 0 + cross_platform = 1 + +enum UserVerificationRequirement: + :u8 + required = 0 + preferred = 1 + discouraged = 2 + +format AuthenticatorSelection: + authenticator_attachment :AuthenticatorAttachment + user_verification :UserVerificationRequirement + +format Challenge: + len :u8 + challenge :[len]u8 + +enum PublicKeyCredentialType: + :u8 + public_key = 0x55 + +enum Trasport: + :u8 + usb = 0x01 + nfc = 0x02 + ble = 0x04 + internal = 0x08 + hybrid = 0x10 + +format PublicKeyCredentialDescriptor: + id_len :u8 + id :[id_len]u8 + type :PublicKeyCredentialType + transports :Trasport # bitfield + +format ExcludeCredentials: + len :u8 + credentials :[len]PublicKeyCredentialDescriptor + +format Extension: + id_len :u8 + id :[id_len]u8 + data_len :u16 + data :[data_len]u8 + +format PublicKeyCredentialParam: + alg :Algorithm + type :PublicKeyCredentialType + +format RelayingParty: + id_len :u8 + id :[id_len]u8 + name_len :u8 + name :[name_len]u8 + +format User: + id_len :u8 + id :[id_len]u8 + name_len :u8 + name :[name_len]u8 + display_name_len :u8 + display_name :[display_name_len]u8 + format CreateInfo: - challenge_len :u8 - challenge :[challenge_len]u8 - \ No newline at end of file + challenge :Challenge + attestation_format_len :u8 + attestation_format :[attestation_format_len]AttestationConveyancePreference + authenticator_selection :AuthenticatorSelection + exclude_credentials :ExcludeCredentials + extensions_len :u8 + extensions :[extensions_len]Extension + pubkey_credential_params_len :u8 + pubkey_credential_params :[pubkey_credential_params_len]PublicKeyCredentialParam + relying_party :RelayingParty + user :User From 6fd5e138f424b2fd6b8b6338207aa3f63b20b52e Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 20:35:33 +0900 Subject: [PATCH 45/50] add ts string convertion of enum --- example/webauthn.bgn | 2 +- src/tool/json2ts/generate.cpp | 92 +++++++++++++++++++++++++++-------- src/tool/json2ts/generate.h | 1 + src/tool/json2ts/main.cpp | 3 ++ 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/example/webauthn.bgn b/example/webauthn.bgn index 4e2feeed..7e3bf388 100644 --- a/example/webauthn.bgn +++ b/example/webauthn.bgn @@ -1,4 +1,4 @@ - +config.url = "https://developer.mozilla.org/ja/docs/Web/API/CredentialsContainer/create#%E3%82%A6%E3%82%A7%E3%83%96%E8%AA%8D%E8%A8%BC_api" format PublicKeyCredential: raw_id_len :u8 raw_id :[raw_id_len]u8 diff --git a/src/tool/json2ts/generate.cpp b/src/tool/json2ts/generate.cpp index 28e983e8..67f3aedd 100644 --- a/src/tool/json2ts/generate.cpp +++ b/src/tool/json2ts/generate.cpp @@ -32,6 +32,7 @@ namespace json2ts { bool typescript = true; bool use_bigint = false; bool no_resize = false; + bool enum_to_string = true; size_t get_seq() { return seq_++; @@ -932,29 +933,81 @@ namespace json2ts { w.writeln("}"); } - void generate(const std::shared_ptr& p) { - for (auto& elem : p->elements) { - if (auto enum_ = ast::as(elem)) { - futils::helper::DynDefer d; + void write_enum(const std::shared_ptr& enum_) { + futils::helper::DynDefer d; + if (typescript) { + w.writeln("export const enum ", enum_->ident->ident, " {"); + d = futils::helper::defer_ex([&] { + w.writeln("}"); + }); + } + { + auto s = w.indent_scope(); + for (auto& elem : enum_->members) { + auto v = str.to_string(elem->value); if (typescript) { - w.writeln("export const enum ", enum_->ident->ident, " {"); - d = futils::helper::defer_ex([&] { - w.writeln("}"); - }); + w.writeln(elem->ident->ident, " = ", v, ","); + str.map_ident(elem->ident, enum_->ident->ident, ".", elem->ident->ident); } - { - auto s = w.indent_scope(); - for (auto& elem : enum_->members) { - auto v = str.to_string(elem->value); - if (typescript) { - w.writeln(elem->ident->ident, " = ", v, ","); - str.map_ident(elem->ident, enum_->ident->ident, ".", elem->ident->ident); - } - else { - str.map_ident(elem->ident, v, "/*", enum_->ident->ident, ".", elem->ident->ident, "*/"); - } + else { + str.map_ident(elem->ident, v, "/*", enum_->ident->ident, ".", elem->ident->ident, "*/"); + } + } + } + if (enum_to_string) { + w.write("export ", enum_->ident->ident, "_to_string(x"); + if (typescript) { + w.write(":", enum_->ident->ident); + } + w.writeln(") {"); + { + auto s = w.indent_scope(); + for (auto& elem : enum_->members) { + auto value = "\"" + brgen::escape(elem->ident->ident) + "\""; + if (elem->str_literal) { + value = elem->str_literal->value; } + auto key = str.to_string(elem->ident); + w.writeln("if(x === ", key, ") {"); + { + auto s = w.indent_scope(); + w.writeln("return ", value, ";"); + } + w.writeln("}"); + } + w.writeln("return null;"); + } + w.writeln("}"); + w.write("export ", enum_->ident->ident, "_from_string(x"); + if (typescript) { + w.write(":", enum_->ident->ident); + } + w.writeln(") {"); + { + auto s = w.indent_scope(); + for (auto& elem : enum_->members) { + auto value = "\"" + brgen::escape(elem->ident->ident) + "\""; + if (elem->str_literal) { + value = elem->str_literal->value; + } + auto key = str.to_string(elem->ident); + w.writeln("if(x === ", value, ") {"); + { + auto s = w.indent_scope(); + w.writeln("return ", key, ";"); + } + w.writeln("}"); } + w.writeln("return null;"); + } + w.writeln("}"); + } + } + + void generate(const std::shared_ptr& p) { + for (auto& elem : p->elements) { + if (auto enum_ = ast::as(elem)) { + write_enum(ast::cast_to(elem)); } } auto s = ast::tool::FormatSorter{}; @@ -970,6 +1023,7 @@ namespace json2ts { g.typescript = !flags.javascript; g.use_bigint = flags.use_bigint; g.no_resize = flags.no_resize; + g.enum_to_string = flags.enum_to_string; g.str.this_access = "obj"; g.str.cast_handler = [](ast::tool::Stringer& s, const std::shared_ptr& c) { return s.to_string(c->expr); diff --git a/src/tool/json2ts/generate.h b/src/tool/json2ts/generate.h index 73fe6099..29dfe64c 100644 --- a/src/tool/json2ts/generate.h +++ b/src/tool/json2ts/generate.h @@ -5,6 +5,7 @@ namespace json2ts { bool use_bigint = false; bool no_resize = false; bool javascript = false; + bool enum_to_string = false; }; std::string generate(const std::shared_ptr& p, Flags flags); } // namespace json2ts diff --git a/src/tool/json2ts/main.cpp b/src/tool/json2ts/main.cpp index 76e53a75..81b5b882 100644 --- a/src/tool/json2ts/main.cpp +++ b/src/tool/json2ts/main.cpp @@ -24,6 +24,7 @@ struct Flags : futils::cmdline::templ::HelpOption { bool no_resize = false; bool no_color = false; bool legacy_file_pass = false; + bool no_enum_to_string = false; void bind(futils::cmdline::option::Context& ctx) { bind_help(ctx); ctx.VarBool(&spec, "s", "spec mode"); @@ -32,6 +33,7 @@ struct Flags : futils::cmdline::templ::HelpOption { ctx.VarBool(&no_resize, "no-resize", "not resize output buffer"); ctx.VarBool(&no_color, "no-color", "no color mode"); ctx.VarBool(&legacy_file_pass, "f,file", "use legacy file pass mode"); + ctx.VarBool(&no_enum_to_string, "no-enum-to-string", "disable enum to string conversion function generation"); } }; @@ -41,6 +43,7 @@ int ts_generate(const Flags& flags, brgen::request::GenerateSource& req, std::sh tsflags.use_bigint = flags.use_bigint; tsflags.no_resize = flags.no_resize; tsflags.javascript = flags.javascript; + tsflags.enum_to_string = !flags.no_enum_to_string; auto out = json2ts::generate(prog, tsflags); send_source(req.id, std::move(out), req.name + (flags.javascript ? ".js" : ".ts")); send_end_response(req.id); From c3ddf09be597168d52a7ac81bea9ba9f8de1e7d7 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 20:40:09 +0900 Subject: [PATCH 46/50] fix --- src/tool/json2ts/generate.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tool/json2ts/generate.cpp b/src/tool/json2ts/generate.cpp index 67f3aedd..4ae33a9a 100644 --- a/src/tool/json2ts/generate.cpp +++ b/src/tool/json2ts/generate.cpp @@ -954,8 +954,9 @@ namespace json2ts { } } } + d.execute(); if (enum_to_string) { - w.write("export ", enum_->ident->ident, "_to_string(x"); + w.write("export function ", enum_->ident->ident, "_to_string(x"); if (typescript) { w.write(":", enum_->ident->ident); } @@ -978,9 +979,9 @@ namespace json2ts { w.writeln("return null;"); } w.writeln("}"); - w.write("export ", enum_->ident->ident, "_from_string(x"); + w.write("export function ", enum_->ident->ident, "_from_string(x"); if (typescript) { - w.write(":", enum_->ident->ident); + w.write(":string"); } w.writeln(") {"); { From 5289e919c53a7fcd67ae53dd713638fafa5137aa Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 20:49:08 +0900 Subject: [PATCH 47/50] fix --- example/webauthn.bgn | 1 + 1 file changed, 1 insertion(+) diff --git a/example/webauthn.bgn b/example/webauthn.bgn index 7e3bf388..c145bb7a 100644 --- a/example/webauthn.bgn +++ b/example/webauthn.bgn @@ -115,6 +115,7 @@ format User: format CreateInfo: challenge :Challenge + attestation :AttestationConveyancePreference attestation_format_len :u8 attestation_format :[attestation_format_len]AttestationConveyancePreference authenticator_selection :AuthenticatorSelection From 18f6437dcc20ccbe517035865e54a7e2e2dcdbb2 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 20:51:07 +0900 Subject: [PATCH 48/50] fix --- example/webauthn.bgn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/webauthn.bgn b/example/webauthn.bgn index c145bb7a..ff615488 100644 --- a/example/webauthn.bgn +++ b/example/webauthn.bgn @@ -117,7 +117,7 @@ format CreateInfo: challenge :Challenge attestation :AttestationConveyancePreference attestation_format_len :u8 - attestation_format :[attestation_format_len]AttestationConveyancePreference + attestation_formats :[attestation_format_len]AttestationFormat authenticator_selection :AuthenticatorSelection exclude_credentials :ExcludeCredentials extensions_len :u8 From f91c098ddd5ca7bb4950fb102c79045cccbaf88c Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Mon, 29 Apr 2024 21:31:48 +0900 Subject: [PATCH 49/50] fix --- example/webauthn.bgn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/webauthn.bgn b/example/webauthn.bgn index ff615488..f64e1b8f 100644 --- a/example/webauthn.bgn +++ b/example/webauthn.bgn @@ -51,7 +51,7 @@ format AttestationFormat: enum AuthenticatorAttachment: :u8 platform = 0 - cross_platform = 1 + cross_platform = 1,"cross-platform" enum UserVerificationRequirement: :u8 From 1e2643996ea100f9c0141c625733a46fb90d8ab1 Mon Sep 17 00:00:00 2001 From: on-keyday <62627905+on-keyday@users.noreply.github.com> Date: Tue, 30 Apr 2024 00:37:47 +0900 Subject: [PATCH 50/50] fix --- example/http2.bgn | 14 ++++++++++++++ src/tool/json2go/generate.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/example/http2.bgn b/example/http2.bgn index bac532d5..3e661d92 100644 --- a/example/http2.bgn +++ b/example/http2.bgn @@ -18,6 +18,20 @@ enum H2Error: http_1_1_required = 0xd transport = 0x100 +enum H2Type: + :u8 + DATA = 0x0 + HEADERS = 0x1 + PRIORITY = 0x2 + RST_STREAM = 0x3 + SETTINGS = 0x4 + PUSH_PROMISE = 0x5 + PING = 0x6 + GOAWAY = 0x7 + WINDOW_UPDATE = 0x8 + CONTINUATION = 0x9 + + format FrameHeader: len :u24 type:u8 diff --git a/src/tool/json2go/generate.go b/src/tool/json2go/generate.go index d3f9b7da..4cfbaf17 100644 --- a/src/tool/json2go/generate.go +++ b/src/tool/json2go/generate.go @@ -521,7 +521,7 @@ func (g *Generator) writeTypeEncode(ident string, typ ast2go.Type, p *ast2go.Fie if arr_type, ok := typ.(*ast2go.ArrayType); ok { if i_typ, ok := arr_type.ElementType.(*ast2go.IntType); ok && *i_typ.BitSize == 8 { if arr_type.Length.GetConstantLevel() == ast2go.ConstantLevelConstant { - g.PrintfFunc("if n,err := w.Write(%s);err != nil || n != len(%s) {\n", ident, ident) + g.PrintfFunc("if n,err := w.Write(%s[:]);err != nil || n != len(%s) {\n", ident, ident) g.imports["fmt"] = struct{}{} g.PrintfFunc("return fmt.Errorf(\"encode %s: %%w\", err)\n", p.Ident.Ident) g.PrintfFunc("}\n")