Merge pull request #1345 from ton-blockchain/tolk-v0.6.0

Tolk Language: next-generation FunC
ton-blockchain · Nov 2, 2024 · 7151ff2 · 7151ff2
2 parents eed3153 + d110022
commit 7151ff2
Show file tree

Hide file tree

Showing 204 changed files with 23,041 additions and 47 deletions.
diff --git a/.github/script/amd64-20.04.Dockerfile b/.github/script/amd64-20.04.Dockerfile
@@ -17,4 +17,4 @@ ENV CC clang
 ENV CXX clang++
 ENV CCACHE_DISABLE 1
 RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
-RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client
+RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func tolk validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client
diff --git a/.github/script/amd64-22.04.Dockerfile b/.github/script/amd64-22.04.Dockerfile
@@ -17,4 +17,4 @@ ENV CC clang
 ENV CXX clang++
 ENV CCACHE_DISABLE 1
 RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
-RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client
+RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func tolk validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client
diff --git a/.github/script/arm64-20.04.Dockerfile b/.github/script/arm64-20.04.Dockerfile
@@ -17,4 +17,4 @@ ENV CC clang
 ENV CXX clang++
 ENV CCACHE_DISABLE 1
 RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
-RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client
+RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func tolk validator-engine validator-engine-console create-state generate-random-id dht-server lite-client
diff --git a/.github/script/arm64-22.04.Dockerfile b/.github/script/arm64-22.04.Dockerfile
@@ -17,4 +17,4 @@ ENV CC clang
 ENV CXX clang++
 ENV CCACHE_DISABLE 1
 RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
-RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client
+RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func tolk validator-engine validator-engine-console create-state generate-random-id dht-server lite-client
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
@@ -167,6 +167,14 @@ jobs:
           asset_name: func.exe
           tag: ${{ steps.tag.outputs.TAG }}
 
+      - name: Upload Windows 2019 single artifact - tolk
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-win-binaries/tolk.exe
+          asset_name: tolk.exe
+          tag: ${{ steps.tag.outputs.TAG }}
+
       - name: Upload Windows 2019 single artifact - lite-client
         uses: svenstaro/upload-release-action@v2
         with:
@@ -257,6 +265,14 @@ jobs:
           asset_name: func-mac-x86-64
           tag: ${{ steps.tag.outputs.TAG }}
 
+      - name: Upload Mac x86-64 single artifact - tolk
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-x86_64-macos-binaries/tolk
+          asset_name: tolk-mac-x86-64
+          tag: ${{ steps.tag.outputs.TAG }}
+
       - name: Upload Mac x86-64 single artifact - lite-client
         uses: svenstaro/upload-release-action@v2
         with:
@@ -348,6 +364,14 @@ jobs:
           asset_name: func-mac-arm64
           tag: ${{ steps.tag.outputs.TAG }}
 
+      - name: Upload Mac arm64 single artifact - tolk
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-arm64-macos-binaries/tolk
+          asset_name: tolk-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
       - name: Upload Mac arm64 single artifact - lite-client
         uses: svenstaro/upload-release-action@v2
         with:
@@ -438,6 +462,14 @@ jobs:
           asset_name: func-linux-x86_64
           tag: ${{ steps.tag.outputs.TAG }}
 
+      - name: Upload Linux x86-64 single artifact - tolk
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-x86_64-linux-binaries/tolk
+          asset_name: tolk-linux-x86_64
+          tag: ${{ steps.tag.outputs.TAG }}
+
       - name: Upload Linux x86-64 single artifact - lite-client
         uses: svenstaro/upload-release-action@v2
         with:

diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml
@@ -29,6 +29,7 @@ jobs:
         artifacts/lite-client -V
         artifacts/fift -V
         artifacts/func -V
+        artifacts/tolk -v
 
     - name: Upload artifacts
       uses: actions/upload-artifact@master

diff --git a/.github/workflows/ton-x86-64-linux.yml b/.github/workflows/ton-x86-64-linux.yml
@@ -33,6 +33,7 @@ jobs:
         artifacts/lite-client -V
         artifacts/fift -V
         artifacts/func -V
+        artifacts/tolk -v
 
     - name: Upload artifacts
       uses: actions/upload-artifact@master

diff --git a/.github/workflows/ton-x86-64-macos.yml b/.github/workflows/ton-x86-64-macos.yml
@@ -29,6 +29,7 @@ jobs:
         artifacts/lite-client -V
         artifacts/fift -V
         artifacts/func -V
+        artifacts/tolk -v
 
     - name: Upload artifacts
       uses: actions/upload-artifact@master

diff --git a/.gitignore b/.gitignore
@@ -13,6 +13,8 @@ test/regression-tests.cache/
 **/*build*/
 .idea
 .vscode
+.DS_Store
+dev/
 zlib/
 libsodium/
 libmicrohttpd-0.9.77-w32-bin/

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -413,6 +413,7 @@ add_subdirectory(adnl)
 add_subdirectory(crypto)
 add_subdirectory(lite-client)
 add_subdirectory(emulator)
+add_subdirectory(tolk)
 
 #BEGIN tonlib
 add_subdirectory(tonlib)
@@ -626,6 +627,30 @@ if (NOT NIX)
     endif()
 endif()
 
+# Tolk tests
+if (NOT NIX)
+    if (MSVC)
+      set(PYTHON_VER "python")
+    else()
+      set(PYTHON_VER "python3")
+    endif()
+    add_test(
+      NAME test-tolk
+      COMMAND ${PYTHON_VER} tolk-tester.py tests/
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tolk-tester)
+    if (WIN32)
+        set_property(TEST test-tolk PROPERTY ENVIRONMENT
+          "TOLK_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/tolk/tolk.exe"
+          "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift.exe"
+          "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/")
+    else()
+        set_property(TEST test-tolk PROPERTY ENVIRONMENT
+          "TOLK_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/tolk/tolk"
+          "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift"
+          "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/")
+    endif()
+endif()
+
 #BEGIN internal
 if (NOT TON_ONLY_TONLIB)
 add_test(test-adnl test-adnl)

diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh
@@ -153,7 +153,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 
 if [ "$with_tests" = true ]; then
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
-  tonlib tonlibjson tonlib-cli validator-engine func fift \
+  tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
   lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \
@@ -162,7 +162,7 @@ if [ "$with_tests" = true ]; then
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
-  tonlib tonlibjson tonlib-cli validator-engine func fift \
+  tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
   lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
@@ -173,6 +173,7 @@ strip -s storage/storage-daemon/storage-daemon-cli
 strip -s blockchain-explorer/blockchain-explorer
 strip -s crypto/fift
 strip -s crypto/func
+strip -s tolk/tolk
 strip -s crypto/create-state
 strip -s crypto/tlbc
 strip -s validator-engine-console/validator-engine-console
@@ -197,6 +198,7 @@ if [ "$with_artifacts" = true ]; then
   cp build/blockchain-explorer/blockchain-explorer artifacts/
   cp build/crypto/fift artifacts/
   cp build/crypto/func artifacts/
+  cp build/tolk/tolk artifacts/
   cp build/crypto/create-state artifacts/
   cp build/crypto/tlbc artifacts/
   cp build/validator-engine-console/validator-engine-console artifacts/

diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh
@@ -81,7 +81,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 
 if [ "$with_tests" = true ]; then
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
-  tonlib tonlibjson tonlib-cli validator-engine func fift \
+  tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
   lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \
@@ -90,7 +90,7 @@ if [ "$with_tests" = true ]; then
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \
-  tonlib tonlibjson tonlib-cli validator-engine func fift \
+  tonlib tonlibjson tonlib-cli validator-engine func tolk fift \
   lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
@@ -102,6 +102,7 @@ strip -s storage/storage-daemon/storage-daemon-cli
 strip -s blockchain-explorer/blockchain-explorer
 strip -s crypto/fift
 strip -s crypto/func
+strip -s tolk/tolk
 strip -s crypto/create-state
 strip -s crypto/tlbc
 strip -s validator-engine-console/validator-engine-console
@@ -126,6 +127,7 @@ if [ "$with_artifacts" = true ]; then
   cp build/blockchain-explorer/blockchain-explorer artifacts/
   cp build/crypto/fift artifacts/
   cp build/crypto/func artifacts/
+  cp build/tolk/tolk artifacts/
   cp build/crypto/create-state artifacts/
   cp build/crypto/tlbc artifacts/
   cp build/validator-engine-console/validator-engine-console artifacts/

diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh
@@ -144,7 +144,7 @@ cmake -GNinja .. \
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 
 if [ "$with_tests" = true ]; then
-ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
+ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \
       generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
       adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \
@@ -153,7 +153,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
       test-fec test-tddb test-db test-validator-session-state test-emulator
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
-ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
+ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \
       generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
       adnl-proxy create-state emulator
@@ -166,6 +166,7 @@ strip -s storage/storage-daemon/storage-daemon \
          crypto/fift \
          crypto/tlbc \
          crypto/func \
+         tolk/tolk \
          crypto/create-state \
          validator-engine-console/validator-engine-console \
          tonlib/tonlib-cli \
@@ -195,7 +196,7 @@ if [ "$with_artifacts" = true ]; then
   mkdir artifacts
   mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so
   cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \
-     build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/blockchain-explorer/blockchain-explorer \
+     build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \
      build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli \
      build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \
      build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \

diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh
@@ -52,7 +52,7 @@ cmake -GNinja -DTON_USE_JEMALLOC=ON .. \
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 
 if [ "$with_tests" = true ]; then
-ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
+ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \
       generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
       adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \
@@ -61,7 +61,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
       test-fec test-tddb test-db test-validator-session-state test-emulator
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
-ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
+ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \
       validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \
       generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \
       adnl-proxy create-state emulator
@@ -74,6 +74,7 @@ strip -s storage/storage-daemon/storage-daemon \
          crypto/fift \
          crypto/tlbc \
          crypto/func \
+         tolk/tolk \
          crypto/create-state \
          validator-engine-console/validator-engine-console \
          tonlib/tonlib-cli \
@@ -105,7 +106,7 @@ if [ "$with_artifacts" = true ]; then
   mkdir artifacts
   mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so
   cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \
-     build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/blockchain-explorer/blockchain-explorer \
+     build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \
      build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli \
      build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \
      build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \

diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat
@@ -155,7 +155,7 @@ IF %errorlevel% NEQ 0 (
 )
 
 IF "%1"=="-t" (
-ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson  ^
+ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson  ^
 tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^
 test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^
@@ -166,7 +166,7 @@ IF %errorlevel% NEQ 0 (
   exit /b %errorlevel%
 )
 ) else (
-ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson  ^
+ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson  ^
 tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator
 IF %errorlevel% NEQ 0 (
@@ -204,6 +204,7 @@ build\blockchain-explorer\blockchain-explorer.exe ^
 build\crypto\fift.exe ^
 build\crypto\tlbc.exe ^
 build\crypto\func.exe ^
+build\tolk\tolk.exe ^
 build\crypto\create-state.exe ^
 build\validator-engine-console\validator-engine-console.exe ^
 build\tonlib\tonlib-cli.exe ^

diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat
@@ -156,7 +156,7 @@ IF %errorlevel% NEQ 0 (
 )
 
 IF "%1"=="-t" (
-ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson  ^
+ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson  ^
 tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^
 test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^
@@ -167,7 +167,7 @@ IF %errorlevel% NEQ 0 (
   exit /b %errorlevel%
 )
 ) else (
-ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson  ^
+ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson  ^
 tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^
 json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator
 IF %errorlevel% NEQ 0 (
@@ -205,6 +205,7 @@ build\blockchain-explorer\blockchain-explorer.exe ^
 build\crypto\fift.exe ^
 build\crypto\tlbc.exe ^
 build\crypto\func.exe ^
+build\tolk\tolk.exe ^
 build\crypto\create-state.exe ^
 build\validator-engine-console\validator-engine-console.exe ^
 build\tonlib\tonlib-cli.exe ^

diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh
@@ -43,6 +43,7 @@ sudo strip -s storage-daemon \
          fift \
          tlbc \
          func \
+         tolk \
          create-state \
          validator-engine-console \
          tonlib-cli \

diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh
@@ -43,6 +43,7 @@ sudo strip -s storage-daemon \
          fift \
          tlbc \
          func \
+         tolk \
          create-state \
          validator-engine-console \
          tonlib-cli \

diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh
@@ -43,6 +43,7 @@ sudo strip -xSX storage-daemon \
          fift \
          tlbc \
          func \
+         tolk \
          create-state \
          validator-engine-console \
          tonlib-cli \

diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif
@@ -1589,6 +1589,9 @@ forget @proclist  forget @proccnt
 { }END> b> } : }END>c
 { }END>c <s } : }END>s
 
+// This is the way how FunC assigns method_id for reserved functions.
+// Note, that Tolk entrypoints have other names (`onInternalMessage`, etc.),
+// but method_id is assigned not by Fift, but by Tolk code generation.
 0 constant recv_internal
 -1 constant recv_external
 -2 constant run_ticktock