生成コードテスト用ツールの作成とCIへの組み込み

.github/workflows/brgen-test.yaml

@@ -26,7 +26,7 @@ jobs:
           path: /tmp/brgen
       - name: Install Dependencies
         run: sudo apt-get install jq libc++-dev ninja-build -y
-      - name: Run Brgen and Test
+      - name: Run Brgen
         run: |
           chmod -R +x /tmp/brgen
           cd /tmp/brgen
@@ -35,9 +35,14 @@ jobs:
           ldd /tmp/brgen/tool/libfutils.so
           /tmp/brgen/tool/src2json --version
           /tmp/brgen/tool/brgen -debug
+      - name: Clone Utils
+        run: |
+          cd /tmp/brgen
           git clone https://github.com/on-keyday/utils.git
-          cd utils
-          . build shared Debug futils
-          cd ..
-          export FUTILS_DIR=/tmp/brgen/utils
-          # ./make_cpp_tests.sh ./ignore/example/test_info.json run
+          mkdir utils/lib
+          cp tool/libfutils.so utils/lib
+      - name: Test
+        run: |
+          cd /tmp/brgen
+          . ./link_path.sh
+          FUTILS_DIR=/tmp/brgen/utils tool/cmptest -f ignore/example/test_info.json -c testkit/cmptest.json

.github/workflows/build.yml

@@ -57,13 +57,30 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Set up Ninja
         uses: seanmiddleditch/gha-setup-ninja@8b297075da4cd2a5f1fd21fe011b499edf06e9d2 # master
+      - name: Set up Go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: ">=1.20"
       - name: Install Homebrew Clang
         run: |
           brew install llvm
+
+      - name: Check Code
+        run: |
+          PREFIX=$(brew --prefix llvm)
+          # from https://github.com/llvm/llvm-project/issues/77786
+          LIBCPP=$PREFIX/lib/c++
+          ls $LIBCPP
+          echo "libc++"
+          nm $LIBCPP/libc++.1.dylib | c++filt
+          echo "libc++abi"
+          nm $LIBCPP/libc++abi.1.dylib | c++filt
       - name: Build
         run: |
-          export FUTILS_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++
-          export FUTILS_C_COMPILER=/usr/local/opt/llvm/bin/clang
+          PREFIX=$(brew --prefix llvm)
+          export FUTILS_CXX_COMPILER=$PREFIX/bin/clang++
+          export FUTILS_C_COMPILER=$PREFIX/bin/clang
+          export LDFLAGS="-L$PREFIX/lib/c++ -Wl,-rpath,$PREFIX/lib/c++"
           . build.sh native Release
           ./tool/src2json --version
       - name: Pack artifacts

.gitignore

@@ -31,3 +31,5 @@ go_dump_env.bat
 license_cache
 pkg/
 stats.json
+zig-cache/
+zig-out/

.vscode/launch.json

@@ -57,19 +57,10 @@
             "name": "Debug json2cpp",
             "type": "cppvsdbg",
             "request": "launch",
-            "program": "${workspaceRoot}/tool/json2cpp2",
+            "program": "${workspaceRoot}/tool/json2ts",
             "args": [
-                "<",
-                "ignore/request.bin",
-                ">",
-                "ignore/response.bin",
-                //"ignore/sample.json",
-                //"-f",
-                /*"-r",
-                "--call", "Test2.decode",
-                "--binary","example/vm_test/varint.dat",
-                "--hex",
-                */
+                "-f",
+                "./ignore/sample.json",
             ],
             "stopAtEntry": false,
             "cwd": "${workspaceRoot}",

.vscode/settings.json

@@ -148,6 +148,7 @@
         ".\\ast2rust\\Cargo.toml",
         ".\\src\\tool\\json2rust\\Cargo.toml",
         ".\\src\\tool\\json2llvm\\Cargo.toml",
+        ".\\src\\tool\\cmptest\\Cargo.toml",
     ],
     "brgen-lsp.src2json" :"./tool/src2json.exe",
     //"go.buildTags": "linux",

CMakeLists.txt

@@ -16,6 +16,9 @@ set(CMAKE_EXE_LINKER_FLAGS "-lc++abi")
 endif()
 endif()
 
+
+
+
 set(CMAKE_CXX_STANDARD 20)
 
 

ast2go/request/response.go

@@ -1,14 +1,32 @@
 package request
 
-import "io"
+import (
+	"fmt"
+	"io"
+	"runtime/debug"
+)
 
 type RequestStream struct {
 	r io.Reader
 	w io.Writer
 }
 
-func NewRequestStream(r io.Reader, w io.Writer) *RequestStream {
-	return &RequestStream{r: r, w: w}
+func NewRequestStream(r io.Reader, w io.Writer) (*RequestStream, error) {
+	h := &RequestStream{r: r, w: w}
+	if err := h.handshake(); err != nil {
+		return nil, err
+	}
+	return h, nil
+}
+
+func (rs *RequestStream) handshake() error {
+	resp := &GeneratorResponseHeader{Version: 1}
+	enc := resp.MustEncode()
+	if _, err := rs.w.Write(enc); err != nil {
+		return err
+	}
+	hdr := &GeneratorRequestHeader{}
+	return hdr.Read(rs.r)
 }
 
 func (rs *RequestStream) Receive() (*GenerateSource, error) {
@@ -20,11 +38,77 @@ func (rs *RequestStream) Receive() (*GenerateSource, error) {
 	return src, nil
 }
 
-func (rs *RequestStream) Respond(src *SourceCode) error {
-	enc, err := src.Encode()
+func (rs *RequestStream) respond(id uint64, status ResponseStatus, name string, code []byte, warn string) error {
+	src := &SourceCode{
+		ID:     id,
+		Status: status,
+	}
+	src.SetName([]byte(name))
+	src.SetCode(code)
+	src.SetErrorMessage([]byte(warn))
+	resp := &Response{
+		Type: ResponseType_Code,
+	}
+	resp.SetSource(*src)
+	enc := resp.MustEncode()
+	_, err := rs.w.Write(enc)
+	return err
+}
+
+func (rs *RequestStream) RespondSource(id uint64, name string, code []byte, warn string) error {
+	return rs.respond(id, ResponseStatus_Ok, name, code, warn)
+}
+
+func (rs *RequestStream) RespondError(id uint64, err string) error {
+	return rs.respond(id, ResponseStatus_Error, "", nil, err)
+}
+
+func (rs *RequestStream) RespondEnd(id uint64) error {
+	resp := &Response{
+		Type: ResponseType_EndOfCode,
+	}
+	resp.SetEnd(EndOfCode{ID: id})
+	enc := resp.MustEncode()
+	_, err := rs.w.Write(enc)
+	return err
+}
+
+type IDStream struct {
+	rs *RequestStream
+	id uint64
+}
+
+func (rs *IDStream) RespondSource(name string, code []byte, warn string) error {
+	return rs.rs.RespondSource(rs.id, name, code, warn)
+}
+
+func (rs *IDStream) RespondError(err string) error {
+	return rs.rs.RespondError(rs.id, err)
+}
+
+func Run(r io.Reader, w io.Writer, cb func(s *IDStream, req *GenerateSource)) error {
+	stream, err := NewRequestStream(r, w)
 	if err != nil {
 		return err
 	}
-	_, err = rs.w.Write(enc)
-	return err
+	for {
+		req, err := stream.Receive()
+		if err != nil {
+			break
+		}
+		go func() {
+			defer stream.RespondEnd(req.ID)
+			defer func() {
+				if r := recover(); r != nil {
+					trace := debug.Stack()
+					stream.RespondError(req.ID, fmt.Sprintf("%v\n%s", r, trace))
+				}
+			}()
+			cb(&IDStream{
+				stream,
+				req.ID,
+			}, req)
+		}()
+	}
+	return nil
 }

brgen.json

@@ -53,7 +53,7 @@
       "args": ["-as-code-block", "-format"]
     },
     {
-      "generator": "./src/tool/json2rust/target/debug/json2rust",
+      "generator": "./tool/json2rust",
       "output_dir": "./ignore/example/rust/",
       "ignore_missing": true
     },

build.bat

@@ -1,6 +1,7 @@
 @echo off
 setlocal
 set S2J_LIB=1
+set BRGEN_RUST_ENABLED=1
 set BUILD_MODE=%1
 set BUILD_TYPE=%2
 set FUTILS_DIR=%3
@@ -41,10 +42,10 @@ rem ninja -C ./built/%BUILD_MODE%/%BUILD_TYPE%
 ninja -C ./built/%BUILD_MODE%/%BUILD_TYPE% install
 
 if "%BUILD_MODE%" == "wasm-em" (
-    cd ./src/tool/json2rust
-    wasm-pack build --target web
-    copy ..\..\..\LICENSE .\pkg\LICENSE
-    cd ../../../
+rem    cd ./src/tool/json2rust
+rem    wasm-pack build --target web
+rem    copy ..\..\..\LICENSE .\pkg\LICENSE
+rem    cd ../../../
     cd ./web/dev
     call tsc
     call webpack

build.sh

@@ -1,9 +1,14 @@
 #!/bin/bash
 # TODO(on-keyday): on macos, S2J_LIB=1 is not work well, so not set S2J_LIB
-if [ "$(uname)" != "Darwin" ]; then
-   export S2J_LIB=1
+if [ "$(uname)" == "Darwin" ]; then
+  export S2J_LIB=0
+else
+  export S2J_LIB=1
 fi
 
+
+export BRGEN_RUST_ENABLED=1
+
 BUILD_MODE=$1
 BUILD_TYPE=$2
 
@@ -48,16 +53,16 @@ if [ $BUILD_MODE = "wasm-em" ];then
 else
    cmake  -D CMAKE_CXX_COMPILER=$CXX_COMPILER -D CMAKE_C_COMPILER=$C_COMPILER -G Ninja -D CMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -D CMAKE_BUILD_TYPE=$BUILD_TYPE -S . -B ./built/$BUILD_MODE/$BUILD_TYPE
 fi
-ninja -C ./built/$BUILD_MODE/$BUILD_TYPE
+#ninja -C ./built/$BUILD_MODE/$BUILD_TYPE
 ninja -C ./built/$BUILD_MODE/$BUILD_TYPE install
 
 if [ $BUILD_MODE = "wasm-em" ]; then
 unset GOOS
 unset GOARCH
-cd ./src/tool/json2rust
-wasm-pack build --target web
-cp ../../../LICENSE ./pkg
-cd ../../..
+#cd ./src/tool/json2rust
+#wasm-pack build --target web
+#cp ../../../LICENSE ./pkg
+#cd ../../..
 cd ./web/dev
 tsc
 webpack
@@ -68,3 +73,4 @@ fi
 unset FUTILS_DIR
 unset BUILD_MODE
 unset S2J_LIB
+unset BRGEN_RUST_ENABLED

example/fs.txt

@@ -426,4 +426,24 @@ enum Type  {
         // others are reserved
         experimental_1 = 253,
         experimental_2 = 255,
+    };
+
+ enum  OS  {
+        fat = 0,
+        amiga = 1,
+        vms = 2,
+        unix_ = 3,
+        vm_cms = 4,
+        atari = 5,
+        hpfs = 6,
+        machintosh = 7,
+        z_system = 8,
+        cp_m = 9,
+        tops_20 = 10,
+        ntfs = 11,
+        qdos = 12,
+        acorn_risc_os = 13,
+        vfat = 14,
+        mvs = 15,
+        beos = 16,
     };

example/gzip.bgn

@@ -0,0 +1,60 @@
+config.url = "https://www.rfc-editor.org/rfc/rfc1952"
+input.endian = config.endian.little
+input.bit_order = config.bit_order.lsb
+
+format GZipHeader:
+    magic :"\x1f\x8b"
+    method :CompressionMethod
+    flags :GZipFlag
+    mtime :u32
+    xfl :u8
+    os :OS
+    if flags.extra ==1:
+        extra :GZipExtra
+    if flags.fname ==1:
+        fname :CString
+    if flags.fcomment ==1:
+        fcomment :CString
+    if flags.crc ==1:
+        crc16 :u16
+
+format GZipExtra:
+    xlen :u16
+    data :[xlen]u8
+
+format CString:
+    data :[..]u8
+    :"\x00"
+
+format GZipFlag:
+    text :u1
+    crc :u1
+    extra :u1
+    fname :u1
+    fcomment :u1
+    reserved :u3
+    reserved == 0
+
+enum CompressionMethod:
+    :u8
+    deflate = 8
+
+enum OS:
+    :u8
+    fat = 0
+    amiga = 1
+    vms = 2
+    unix_ = 3
+    vm_cms = 4
+    atari = 5
+    hpfs = 6
+    machintosh = 7
+    z_system = 8
+    cp_m = 9
+    tops_20 = 10
+    ntfs = 11
+    qdos = 12
+    acorn_risc_os = 13
+    vfat = 14
+    mvs = 15
+    beos = 16